From 38f8ec1d7150d4495763ad9880d04aebc0314966 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 09:30:48 +0800 Subject: [PATCH 01/16] chore(deps): Bump uuid from 1.4.1 to 1.5.0 (#12955) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a337219231016..94d936ed0c51a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10479,9 +10479,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom", "rand", From fdc4139f1db60544cf0c24b33984c9babbcdb41b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 09:30:51 +0800 Subject: [PATCH 02/16] chore(deps): Bump rustix from 0.36.15 to 0.36.16 (#12953) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94d936ed0c51a..bb58e6cac3fa1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6208,7 +6208,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.15", + "rustix 0.36.16", ] [[package]] @@ -6221,7 +6221,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.15", + "rustix 0.36.16", ] [[package]] @@ -8288,9 +8288,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.15" +version = "0.36.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ "bitflags 1.3.2", "errno", From 8d5d518e5a2994a0da4b6b045542816d67153e01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 09:30:54 +0800 Subject: [PATCH 03/16] chore(deps): Bump rustix from 0.38.4 to 0.38.19 in /src/tests/libpq_test (#12954) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- src/tests/libpq_test/Cargo.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tests/libpq_test/Cargo.lock b/src/tests/libpq_test/Cargo.lock index 38710822ca506..74beef901a9ea 100644 --- a/src/tests/libpq_test/Cargo.lock +++ b/src/tests/libpq_test/Cargo.lock @@ -96,9 +96,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "cc" @@ -249,9 +249,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libloading" @@ -288,9 +288,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.3" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "log" @@ -402,11 +402,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.4" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", From 1b0d5eeef86cc2dca8a4ced86b7f92fe706194f9 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Wed, 18 Oct 2023 21:29:46 -0500 Subject: [PATCH 04/16] fix(stream): add assertion to ensure pk indices consistency (#12763) Signed-off-by: Richard Chien --- src/stream/src/from_proto/source/trad_source.rs | 11 +++++------ src/stream/src/task/stream_manager.rs | 10 ++++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/stream/src/from_proto/source/trad_source.rs b/src/stream/src/from_proto/source/trad_source.rs index 24b77af41e0f6..3f0793595c7c5 100644 --- a/src/stream/src/from_proto/source/trad_source.rs +++ b/src/stream/src/from_proto/source/trad_source.rs @@ -162,13 +162,12 @@ impl ExecutorBuilder for SourceExecutorBuilder { let table_type = ExternalTableType::from_properties(&source.properties); if table_type.can_backfill() && let Some(table_desc) = source_info.upstream_table.clone() { let upstream_table_name = SchemaTableName::from_properties(&source.properties); - let pk_indices = table_desc + let table_pk_indices = table_desc .pk .iter() .map(|k| k.column_index as usize) .collect_vec(); - - let order_types = table_desc + let table_pk_order_types = table_desc .pk .iter() .map(|desc| OrderType::from_protobuf(desc.get_order_type().unwrap())) @@ -180,8 +179,8 @@ impl ExecutorBuilder for SourceExecutorBuilder { upstream_table_name, table_reader, schema.clone(), - order_types, - pk_indices.clone(), + table_pk_order_types, + table_pk_indices, (0..table_desc.columns.len()).collect_vec(), ); @@ -197,7 +196,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { (0..source.columns.len()).collect_vec(), // eliminate the last column (_rw_offset) None, schema.clone(), - pk_indices, + params.pk_indices, params.executor_stats, source_state_handler, source_ctrl_opts.chunk_size diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs index d1afaeb061809..f54eb9921f77c 100644 --- a/src/stream/src/task/stream_manager.rs +++ b/src/stream/src/task/stream_manager.rs @@ -568,10 +568,10 @@ impl LocalStreamManagerCore { // Build the executor with params. let executor_params = ExecutorParams { env: env.clone(), - pk_indices, + pk_indices: pk_indices.clone(), executor_id, operator_id, - identity, + identity: identity.clone(), op_info, schema, input, @@ -583,6 +583,12 @@ impl LocalStreamManagerCore { }; let executor = create_executor(executor_params, self, node, store).await?; + assert_eq!( + executor.pk_indices(), + &pk_indices, + "`pk_indices` of {} not consistent with what derived by optimizer", + executor.identity() + ); // Wrap the executor for debug purpose. let executor = WrapperExecutor::new( From 495ab39ddff44ac70edfd785e0f2ed267e16923b Mon Sep 17 00:00:00 2001 From: wu Date: Thu, 19 Oct 2023 11:41:59 +0800 Subject: [PATCH 05/16] feat(sink): kafka upsert sink with schema (#12113) Co-authored-by: Rossil <40714231+Rossil2012@users.noreply.github.com> --- ci/scripts/e2e-kafka-sink-test.sh | 28 +++ e2e_test/sink/kafka/create_sink.slt | 10 + e2e_test/sink/kafka/drop_sink.slt | 3 + e2e_test/sink/kafka/upsert_schema1.result | 10 + e2e_test/sink/kafka/upsert_schema2.result | 11 + e2e_test/sink/kafka/upsert_schema3.result | 12 ++ src/connector/src/sink/encoder/json.rs | 239 +++++++++++++++++++++- src/connector/src/sink/encoder/mod.rs | 8 + src/connector/src/sink/formatter/mod.rs | 24 ++- 9 files changed, 338 insertions(+), 7 deletions(-) create mode 100644 e2e_test/sink/kafka/upsert_schema1.result create mode 100644 e2e_test/sink/kafka/upsert_schema2.result create mode 100644 e2e_test/sink/kafka/upsert_schema3.result diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh index 85aad20749d61..1e01903ec2042 100755 --- a/ci/scripts/e2e-kafka-sink-test.sh +++ b/ci/scripts/e2e-kafka-sink-test.sh @@ -5,6 +5,7 @@ set -euo pipefail ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --create > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/create_sink.slt' @@ -28,6 +29,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema +echo "testing upsert kafka sink with schema" +diff ./e2e_test/sink/kafka/upsert_schema1.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink echo "testing debezium kafka sink" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null @@ -62,6 +72,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema after update +echo "testing upsert kafka sink with schema after updating data" +diff ./e2e_test/sink/kafka/upsert_schema2.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after update echo "testing debezium kafka sink after updating data" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 11 | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null @@ -87,6 +106,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema after delete +echo "testing upsert kafka sink with schema after deleting data" +diff ./e2e_test/sink/kafka/upsert_schema3.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after delete echo "testing debezium kafka sink after deleting data" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt index 2c938cd2219b6..25e3a59fdff3a 100644 --- a/e2e_test/sink/kafka/create_sink.slt +++ b/e2e_test/sink/kafka/create_sink.slt @@ -97,6 +97,16 @@ create sink si_kafka_upsert from t_kafka with ( primary_key = 'id', ); +statement ok +create sink si_kafka_upsert_schema from t_kafka with ( + connector = 'kafka', + properties.bootstrap.server = '127.0.0.1:29092', + topic = 'test-rw-sink-upsert-schema', + primary_key = 'id', +) format upsert encode json ( + schemas.enable = true +); + statement ok create sink si_kafka_debezium from t_kafka with ( connector = 'kafka', diff --git a/e2e_test/sink/kafka/drop_sink.slt b/e2e_test/sink/kafka/drop_sink.slt index 4b9b10eceb3d7..9479671b9f0f0 100644 --- a/e2e_test/sink/kafka/drop_sink.slt +++ b/e2e_test/sink/kafka/drop_sink.slt @@ -7,5 +7,8 @@ drop sink si_kafka_upsert; statement ok drop sink si_kafka_debezium; +statement ok +drop sink si_kafka_upsert_schema; + statement ok drop table t_kafka; diff --git a/e2e_test/sink/kafka/upsert_schema1.result b/e2e_test/sink/kafka/upsert_schema1.result new file mode 100644 index 0000000000000..d33394de4e5b9 --- /dev/null +++ b/e2e_test/sink/kafka/upsert_schema1.result @@ -0,0 +1,10 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema2.result b/e2e_test/sink/kafka/upsert_schema2.result new file mode 100644 index 0000000000000..e560df682107e --- /dev/null +++ b/e2e_test/sink/kafka/upsert_schema2.result @@ -0,0 +1,11 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema3.result b/e2e_test/sink/kafka/upsert_schema3.result new file mode 100644 index 0000000000000..85c3db9768e78 --- /dev/null +++ b/e2e_test/sink/kafka/upsert_schema3.result @@ -0,0 +1,12 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 25d142ad8357b..12176b491d6ab 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -13,11 +13,13 @@ // limitations under the License. use std::collections::HashMap; +use std::sync::Arc; use base64::engine::general_purpose; use base64::Engine as _; use chrono::{Datelike, Timelike}; use indexmap::IndexMap; +use itertools::Itertools; use risingwave_common::array::{ArrayError, ArrayResult}; use risingwave_common::catalog::{Field, Schema}; use risingwave_common::row::Row; @@ -25,7 +27,10 @@ use risingwave_common::types::{DataType, DatumRef, Decimal, ScalarRefImpl, ToTex use risingwave_common::util::iter_util::ZipEqDebug; use serde_json::{json, Map, Value}; -use super::{CustomJsonType, Result, RowEncoder, SerTo, TimestampHandlingMode}; +use super::{ + CustomJsonType, KafkaConnectParams, KafkaConnectParamsRef, Result, RowEncoder, SerTo, + TimestampHandlingMode, +}; use crate::sink::SinkError; pub struct JsonEncoder { @@ -33,6 +38,7 @@ pub struct JsonEncoder { col_indices: Option>, timestamp_handling_mode: TimestampHandlingMode, custom_json_type: CustomJsonType, + kafka_connect: Option, } impl JsonEncoder { @@ -46,6 +52,7 @@ impl JsonEncoder { col_indices, timestamp_handling_mode, custom_json_type: CustomJsonType::None, + kafka_connect: None, } } @@ -60,6 +67,14 @@ impl JsonEncoder { col_indices, timestamp_handling_mode, custom_json_type: CustomJsonType::Doris(map), + kafka_connect: None, + } + } + + pub fn with_kafka_connect(self, kafka_connect: KafkaConnectParams) -> Self { + Self { + kafka_connect: Some(Arc::new(kafka_connect)), + ..self } } } @@ -81,19 +96,29 @@ impl RowEncoder for JsonEncoder { col_indices: impl Iterator, ) -> Result { let mut mappings = Map::with_capacity(self.schema.len()); - for idx in col_indices { - let field = &self.schema[idx]; + let col_indices = col_indices.collect_vec(); + for idx in &col_indices { + let field = &self.schema[*idx]; let key = field.name.clone(); let value = datum_to_json_object( field, - row.datum_at(idx), + row.datum_at(*idx), self.timestamp_handling_mode, &self.custom_json_type, ) .map_err(|e| SinkError::Encode(e.to_string()))?; mappings.insert(key, value); } - Ok(mappings) + + Ok(if let Some(param) = &self.kafka_connect { + json_converter_with_schema( + Value::Object(mappings), + param.schema_name.to_owned(), + col_indices.into_iter().map(|i| &self.schema[i]), + ) + } else { + mappings + }) } } @@ -262,6 +287,79 @@ fn datum_to_json_object( Ok(value) } +fn json_converter_with_schema<'a>( + object: Value, + name: String, + fields: impl Iterator, +) -> Map { + let mut mapping = Map::with_capacity(2); + mapping.insert( + "schema".to_string(), + json!({ + "type": "struct", + "fields": fields.map(|field| { + let mut mapping = type_as_json_schema(&field.data_type); + mapping.insert("field".to_string(), json!(field.name)); + mapping + }).collect_vec(), + "optional": false, + "name": name, + }), + ); + mapping.insert("payload".to_string(), object); + mapping +} + +// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 +pub(crate) fn schema_type_mapping(rw_type: &DataType) -> &'static str { + match rw_type { + DataType::Boolean => "boolean", + DataType::Int16 => "int16", + DataType::Int32 => "int32", + DataType::Int64 => "int64", + DataType::Float32 => "float", + DataType::Float64 => "double", + DataType::Decimal => "string", + DataType::Date => "int32", + DataType::Varchar => "string", + DataType::Time => "int64", + DataType::Timestamp => "int64", + DataType::Timestamptz => "string", + DataType::Interval => "string", + DataType::Struct(_) => "struct", + DataType::List(_) => "array", + DataType::Bytea => "bytes", + DataType::Jsonb => "string", + DataType::Serial => "int32", + DataType::Int256 => "string", + } +} + +fn type_as_json_schema(rw_type: &DataType) -> Map { + let mut mapping = Map::with_capacity(4); // type + optional + fields/items + field + mapping.insert("type".to_string(), json!(schema_type_mapping(rw_type))); + mapping.insert("optional".to_string(), json!(true)); + match rw_type { + DataType::Struct(struct_type) => { + let sub_fields = struct_type + .iter() + .map(|(sub_name, sub_type)| { + let mut sub_mapping = type_as_json_schema(sub_type); + sub_mapping.insert("field".to_string(), json!(sub_name)); + sub_mapping + }) + .collect_vec(); + mapping.insert("fields".to_string(), json!(sub_fields)); + } + DataType::List(sub_type) => { + mapping.insert("items".to_string(), json!(type_as_json_schema(sub_type))); + } + _ => {} + } + + mapping +} + #[cfg(test)] mod tests { @@ -443,4 +541,135 @@ mod tests { .unwrap(); assert_eq!(interval_value, json!("{\"v3\":3,\"v2\":2,\"v1\":1}")); } + + #[test] + fn test_generate_json_converter_schema() { + let mock_field = Field { + data_type: DataType::Boolean, + name: Default::default(), + sub_fields: Default::default(), + type_name: Default::default(), + }; + let fields = vec![ + Field { + data_type: DataType::Boolean, + name: "v1".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int16, + name: "v2".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int32, + name: "v3".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Float32, + name: "v4".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Decimal, + name: "v5".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Date, + name: "v6".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Varchar, + name: "v7".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Time, + name: "v8".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Interval, + name: "v9".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Struct(StructType::new(vec![ + ("a", DataType::Timestamp), + ("b", DataType::Timestamptz), + ( + "c", + DataType::Struct(StructType::new(vec![ + ("aa", DataType::Int64), + ("bb", DataType::Float64), + ])), + ), + ])), + name: "v10".into(), + sub_fields: vec![ + Field { + data_type: DataType::Timestamp, + name: "a".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Timestamptz, + name: "b".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Struct(StructType::new(vec![ + ("aa", DataType::Int64), + ("bb", DataType::Float64), + ])), + name: "c".into(), + sub_fields: vec![ + Field { + data_type: DataType::Int64, + name: "aa".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Float64, + name: "bb".into(), + ..mock_field.clone() + }, + ], + ..mock_field.clone() + }, + ], + ..mock_field.clone() + }, + Field { + data_type: DataType::List(Box::new(DataType::List(Box::new(DataType::Struct( + StructType::new(vec![("aa", DataType::Int64), ("bb", DataType::Float64)]), + ))))), + name: "v11".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Jsonb, + name: "12".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Serial, + name: "13".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int256, + name: "14".into(), + ..mock_field.clone() + }, + ]; + let schema = json_converter_with_schema(json!({}), "test".to_owned(), fields.iter()) + ["schema"] + .to_string(); + let ans = r#"{"fields":[{"field":"v1","optional":true,"type":"boolean"},{"field":"v2","optional":true,"type":"int16"},{"field":"v3","optional":true,"type":"int32"},{"field":"v4","optional":true,"type":"float"},{"field":"v5","optional":true,"type":"string"},{"field":"v6","optional":true,"type":"int32"},{"field":"v7","optional":true,"type":"string"},{"field":"v8","optional":true,"type":"int64"},{"field":"v9","optional":true,"type":"string"},{"field":"v10","fields":[{"field":"a","optional":true,"type":"int64"},{"field":"b","optional":true,"type":"string"},{"field":"c","fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"}],"optional":true,"type":"struct"},{"field":"v11","items":{"items":{"fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"},"optional":true,"type":"array"},"optional":true,"type":"array"},{"field":"12","optional":true,"type":"string"},{"field":"13","optional":true,"type":"int32"},{"field":"14","optional":true,"type":"string"}],"name":"test","optional":false,"type":"struct"}"#; + assert_eq!(schema, ans); + } } diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index b0a13606948dd..6a2558f5fc35a 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::sync::Arc; use risingwave_common::catalog::Schema; use risingwave_common::row::Row; @@ -131,3 +132,10 @@ impl From for super::SinkError { Self::Encode(value.to_string()) } } + +#[derive(Clone)] +pub struct KafkaConnectParams { + pub schema_name: String, +} + +type KafkaConnectParamsRef = Arc; diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 61fe7a830c0a2..44c8ed6f3ea29 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -27,6 +27,7 @@ use risingwave_common::catalog::Schema; pub use upsert::UpsertFormatter; use super::catalog::{SinkEncode, SinkFormat, SinkFormatDesc}; +use super::encoder::KafkaConnectParams; use crate::sink::encoder::{JsonEncoder, TimestampHandlingMode}; /// Transforms a `StreamChunk` into a sequence of key-value pairs according a specific format, @@ -106,12 +107,31 @@ impl SinkFormatterImpl { ))) } SinkFormat::Upsert => { - let key_encoder = JsonEncoder::new( + let mut key_encoder = JsonEncoder::new( schema.clone(), Some(pk_indices), TimestampHandlingMode::Milli, ); - let val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); + let mut val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); + + if let Some(s) = format_desc.options.get("schemas.enable") { + match s.to_lowercase().parse::() { + Ok(true) => { + let kafka_connect = KafkaConnectParams { + schema_name: format!("{}.{}", db_name, sink_from_name), + }; + key_encoder = key_encoder.with_kafka_connect(kafka_connect.clone()); + val_encoder = val_encoder.with_kafka_connect(kafka_connect); + } + Ok(false) => {} + _ => { + return Err(SinkError::Config(anyhow!( + "schemas.enable is expected to be `true` or `false`, got {}", + s + ))); + } + } + }; // Initialize the upsert_stream let formatter = UpsertFormatter::new(key_encoder, val_encoder); From bf3dbbbdf426f43031f3186cf3447b75d8d7ae66 Mon Sep 17 00:00:00 2001 From: TennyZhuang Date: Thu, 19 Oct 2023 21:40:42 +0800 Subject: [PATCH 06/16] refactor(frontend/expr): use associated type for ExprVisitor design (#12960) Signed-off-by: TennyZhuang --- src/frontend/src/expr/expr_visitor.rs | 47 ++++++++++--------- src/frontend/src/expr/mod.rs | 16 +++++-- src/frontend/src/expr/order_by_expr.rs | 2 +- src/frontend/src/expr/pure.rs | 4 +- src/frontend/src/expr/utils.rs | 8 +++- .../plan_expr_visitor/expr_counter.rs | 4 +- .../plan_expr_visitor/input_ref_counter.rs | 4 +- .../plan_node/logical_over_window.rs | 4 +- .../src/optimizer/plan_node/logical_scan.rs | 4 +- .../plan_visitor/input_ref_validator.rs | 4 +- .../plan_visitor/plan_correlated_id_finder.rs | 4 +- .../optimizer/rule/index_selection_rule.rs | 8 +++- .../optimizer/rule/rewrite_like_expr_rule.rs | 4 +- src/frontend/src/utils/condition.rs | 2 +- 14 files changed, 76 insertions(+), 39 deletions(-) diff --git a/src/frontend/src/expr/expr_visitor.rs b/src/frontend/src/expr/expr_visitor.rs index 4fdfbb07f7518..5bc827b43aba8 100644 --- a/src/frontend/src/expr/expr_visitor.rs +++ b/src/frontend/src/expr/expr_visitor.rs @@ -25,13 +25,15 @@ use super::{ /// /// Note: The default implementation for `visit_subquery` is a no-op, i.e., expressions inside /// subqueries are not traversed. -pub trait ExprVisitor { +pub trait ExprVisitor { + type Result: Default; + /// This merge function is used to reduce results of expr inputs. /// In order to always remind users to implement themselves, we don't provide an default /// implementation. - fn merge(a: R, b: R) -> R; + fn merge(a: Self::Result, b: Self::Result) -> Self::Result; - fn visit_expr(&mut self, expr: &ExprImpl) -> R { + fn visit_expr(&mut self, expr: &ExprImpl) -> Self::Result { match expr { ExprImpl::InputRef(inner) => self.visit_input_ref(inner), ExprImpl::Literal(inner) => self.visit_literal(inner), @@ -47,7 +49,7 @@ pub trait ExprVisitor { ExprImpl::Now(inner) => self.visit_now(inner), } } - fn visit_function_call(&mut self, func_call: &FunctionCall) -> R { + fn visit_function_call(&mut self, func_call: &FunctionCall) -> Self::Result { func_call .inputs() .iter() @@ -55,10 +57,13 @@ pub trait ExprVisitor { .reduce(Self::merge) .unwrap_or_default() } - fn visit_function_call_with_lambda(&mut self, func_call: &FunctionCallWithLambda) -> R { + fn visit_function_call_with_lambda( + &mut self, + func_call: &FunctionCallWithLambda, + ) -> Self::Result { self.visit_function_call(func_call.base()) } - fn visit_agg_call(&mut self, agg_call: &AggCall) -> R { + fn visit_agg_call(&mut self, agg_call: &AggCall) -> Self::Result { let mut r = agg_call .args() .iter() @@ -69,22 +74,22 @@ pub trait ExprVisitor { r = Self::merge(r, agg_call.filter().visit_expr(self)); r } - fn visit_parameter(&mut self, _: &Parameter) -> R { - R::default() + fn visit_parameter(&mut self, _: &Parameter) -> Self::Result { + Self::Result::default() } - fn visit_literal(&mut self, _: &Literal) -> R { - R::default() + fn visit_literal(&mut self, _: &Literal) -> Self::Result { + Self::Result::default() } - fn visit_input_ref(&mut self, _: &InputRef) -> R { - R::default() + fn visit_input_ref(&mut self, _: &InputRef) -> Self::Result { + Self::Result::default() } - fn visit_subquery(&mut self, _: &Subquery) -> R { - R::default() + fn visit_subquery(&mut self, _: &Subquery) -> Self::Result { + Self::Result::default() } - fn visit_correlated_input_ref(&mut self, _: &CorrelatedInputRef) -> R { - R::default() + fn visit_correlated_input_ref(&mut self, _: &CorrelatedInputRef) -> Self::Result { + Self::Result::default() } - fn visit_table_function(&mut self, func_call: &TableFunction) -> R { + fn visit_table_function(&mut self, func_call: &TableFunction) -> Self::Result { func_call .args .iter() @@ -92,7 +97,7 @@ pub trait ExprVisitor { .reduce(Self::merge) .unwrap_or_default() } - fn visit_window_function(&mut self, func_call: &WindowFunction) -> R { + fn visit_window_function(&mut self, func_call: &WindowFunction) -> Self::Result { func_call .args .iter() @@ -100,7 +105,7 @@ pub trait ExprVisitor { .reduce(Self::merge) .unwrap_or_default() } - fn visit_user_defined_function(&mut self, func_call: &UserDefinedFunction) -> R { + fn visit_user_defined_function(&mut self, func_call: &UserDefinedFunction) -> Self::Result { func_call .args .iter() @@ -108,7 +113,7 @@ pub trait ExprVisitor { .reduce(Self::merge) .unwrap_or_default() } - fn visit_now(&mut self, _: &Now) -> R { - R::default() + fn visit_now(&mut self, _: &Now) -> Self::Result { + Self::Result::default() } } diff --git a/src/frontend/src/expr/mod.rs b/src/frontend/src/expr/mod.rs index ba36ad0514d28..6eec2983f5c91 100644 --- a/src/frontend/src/expr/mod.rs +++ b/src/frontend/src/expr/mod.rs @@ -351,7 +351,9 @@ macro_rules! impl_has_variant { pub fn [](&self) -> bool { struct Has {} - impl ExprVisitor for Has { + impl ExprVisitor for Has { + + type Result = bool; fn merge(a: bool, b: bool) -> bool { a | b @@ -422,7 +424,9 @@ impl ExprImpl { depth: usize, } - impl ExprVisitor for Has { + impl ExprVisitor for Has { + type Result = bool; + fn merge(a: bool, b: bool) -> bool { a | b } @@ -480,7 +484,9 @@ impl ExprImpl { correlated_id: CorrelatedId, } - impl ExprVisitor for Has { + impl ExprVisitor for Has { + type Result = bool; + fn merge(a: bool, b: bool) -> bool { a | b } @@ -600,7 +606,9 @@ impl ExprImpl { struct HasOthers { has_others: bool, } - impl ExprVisitor<()> for HasOthers { + impl ExprVisitor for HasOthers { + type Result = (); + fn merge(_: (), _: ()) {} fn visit_expr(&mut self, expr: &ExprImpl) { diff --git a/src/frontend/src/expr/order_by_expr.rs b/src/frontend/src/expr/order_by_expr.rs index 48790ceb39337..e7fe005256640 100644 --- a/src/frontend/src/expr/order_by_expr.rs +++ b/src/frontend/src/expr/order_by_expr.rs @@ -71,7 +71,7 @@ impl OrderBy { } } - pub fn visit_expr + ?Sized>(&self, visitor: &mut V) -> R { + pub fn visit_expr(&self, visitor: &mut V) -> V::Result { self.sort_exprs .iter() .map(|expr| visitor.visit_expr(&expr.expr)) diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs index 44300073fd678..e5d698c2ce172 100644 --- a/src/frontend/src/expr/pure.rs +++ b/src/frontend/src/expr/pure.rs @@ -18,7 +18,9 @@ use super::{ExprImpl, ExprVisitor}; use crate::expr::FunctionCall; pub(crate) struct ImpureAnalyzer {} -impl ExprVisitor for ImpureAnalyzer { +impl ExprVisitor for ImpureAnalyzer { + type Result = bool; + fn merge(a: bool, b: bool) -> bool { // the expr will be impure if any of its input is impure a || b diff --git a/src/frontend/src/expr/utils.rs b/src/frontend/src/expr/utils.rs index d07287b08dbe2..39064d1680359 100644 --- a/src/frontend/src/expr/utils.rs +++ b/src/frontend/src/expr/utils.rs @@ -353,7 +353,9 @@ pub struct CollectInputRef { input_bits: FixedBitSet, } -impl ExprVisitor<()> for CollectInputRef { +impl ExprVisitor for CollectInputRef { + type Result = (); + fn merge(_: (), _: ()) {} fn visit_input_ref(&mut self, expr: &InputRef) { @@ -408,7 +410,9 @@ pub fn collect_input_refs<'a>( #[derive(Clone, Default)] pub struct CountNow {} -impl ExprVisitor for CountNow { +impl ExprVisitor for CountNow { + type Result = usize; + fn merge(a: usize, b: usize) -> usize { a + b } diff --git a/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs b/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs index 3bddc479910c2..c664016b779da 100644 --- a/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs +++ b/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs @@ -23,7 +23,9 @@ pub struct CseExprCounter { pub counter: HashMap, } -impl ExprVisitor<()> for CseExprCounter { +impl ExprVisitor for CseExprCounter { + type Result = (); + fn merge(_: (), _: ()) {} fn visit_expr(&mut self, expr: &ExprImpl) { diff --git a/src/frontend/src/optimizer/plan_expr_visitor/input_ref_counter.rs b/src/frontend/src/optimizer/plan_expr_visitor/input_ref_counter.rs index 45cdeaab55a78..382dc74222c9f 100644 --- a/src/frontend/src/optimizer/plan_expr_visitor/input_ref_counter.rs +++ b/src/frontend/src/optimizer/plan_expr_visitor/input_ref_counter.rs @@ -22,7 +22,9 @@ pub struct InputRefCounter { pub counter: HashMap, } -impl ExprVisitor<()> for InputRefCounter { +impl ExprVisitor for InputRefCounter { + type Result = (); + fn merge(_: (), _: ()) {} fn visit_input_ref(&mut self, input_ref: &InputRef) { diff --git a/src/frontend/src/optimizer/plan_node/logical_over_window.rs b/src/frontend/src/optimizer/plan_node/logical_over_window.rs index c1984efbcfcde..5efe0dac01602 100644 --- a/src/frontend/src/optimizer/plan_node/logical_over_window.rs +++ b/src/frontend/src/optimizer/plan_node/logical_over_window.rs @@ -341,7 +341,9 @@ impl<'a> OverWindowProjectBuilder<'a> { } } -impl<'a> ExprVisitor<()> for OverWindowProjectBuilder<'a> { +impl<'a> ExprVisitor for OverWindowProjectBuilder<'a> { + type Result = (); + fn merge(_a: (), _b: ()) {} fn visit_window_function(&mut self, window_function: &WindowFunction) { diff --git a/src/frontend/src/optimizer/plan_node/logical_scan.rs b/src/frontend/src/optimizer/plan_node/logical_scan.rs index 6307558882023..a499a9c6ea3d3 100644 --- a/src/frontend/src/optimizer/plan_node/logical_scan.rs +++ b/src/frontend/src/optimizer/plan_node/logical_scan.rs @@ -397,7 +397,9 @@ impl PredicatePushdown for LogicalScan { // If the predicate contains `CorrelatedInputRef` or `now()`. We don't push down. // This case could come from the predicate push down before the subquery unnesting. struct HasCorrelated {} - impl ExprVisitor for HasCorrelated { + impl ExprVisitor for HasCorrelated { + type Result = bool; + fn merge(a: bool, b: bool) -> bool { a | b } diff --git a/src/frontend/src/optimizer/plan_visitor/input_ref_validator.rs b/src/frontend/src/optimizer/plan_visitor/input_ref_validator.rs index 48bf5bd7b640c..a8e48633d6a31 100644 --- a/src/frontend/src/optimizer/plan_visitor/input_ref_validator.rs +++ b/src/frontend/src/optimizer/plan_visitor/input_ref_validator.rs @@ -25,7 +25,9 @@ struct ExprVis<'a> { schema: &'a Schema, } -impl ExprVisitor> for ExprVis<'_> { +impl ExprVisitor for ExprVis<'_> { + type Result = Option; + fn visit_input_ref(&mut self, input_ref: &crate::expr::InputRef) -> Option { if input_ref.data_type != self.schema[input_ref.index].data_type { Some(format!( diff --git a/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs b/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs index db3ef0b7acfa4..e71497326a29a 100644 --- a/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs +++ b/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs @@ -129,7 +129,9 @@ impl ExprCorrelatedIdFinder { } } -impl ExprVisitor<()> for ExprCorrelatedIdFinder { +impl ExprVisitor for ExprCorrelatedIdFinder { + type Result = (); + fn merge(_: (), _: ()) {} fn visit_correlated_input_ref(&mut self, correlated_input_ref: &CorrelatedInputRef) { diff --git a/src/frontend/src/optimizer/rule/index_selection_rule.rs b/src/frontend/src/optimizer/rule/index_selection_rule.rs index c16cd7e31bf28..9103d1bc906bc 100644 --- a/src/frontend/src/optimizer/rule/index_selection_rule.rs +++ b/src/frontend/src/optimizer/rule/index_selection_rule.rs @@ -917,7 +917,9 @@ impl IndexCost { } } -impl ExprVisitor for TableScanIoEstimator<'_> { +impl ExprVisitor for TableScanIoEstimator<'_> { + type Result = IndexCost; + fn visit_function_call(&mut self, func_call: &FunctionCall) -> IndexCost { match func_call.func_type() { ExprType::Or => func_call @@ -944,7 +946,9 @@ struct ExprInputRefFinder { pub input_ref_index_set: HashSet, } -impl ExprVisitor<()> for ExprInputRefFinder { +impl ExprVisitor for ExprInputRefFinder { + type Result = (); + fn merge(_: (), _: ()) {} fn visit_input_ref(&mut self, input_ref: &InputRef) { diff --git a/src/frontend/src/optimizer/rule/rewrite_like_expr_rule.rs b/src/frontend/src/optimizer/rule/rewrite_like_expr_rule.rs index 394d569050c27..facad4a8da07c 100644 --- a/src/frontend/src/optimizer/rule/rewrite_like_expr_rule.rs +++ b/src/frontend/src/optimizer/rule/rewrite_like_expr_rule.rs @@ -48,7 +48,9 @@ impl Rule for RewriteLikeExprRule { struct HasLikeExprVisitor {} -impl ExprVisitor for HasLikeExprVisitor { +impl ExprVisitor for HasLikeExprVisitor { + type Result = bool; + fn merge(a: bool, b: bool) -> bool { a | b } diff --git a/src/frontend/src/utils/condition.rs b/src/frontend/src/utils/condition.rs index 57910083dfbaf..5305f9f1f356a 100644 --- a/src/frontend/src/utils/condition.rs +++ b/src/frontend/src/utils/condition.rs @@ -844,7 +844,7 @@ impl Condition { .simplify() } - pub fn visit_expr + ?Sized>(&self, visitor: &mut V) -> R { + pub fn visit_expr(&self, visitor: &mut V) -> V::Result { self.conjunctions .iter() .map(|expr| visitor.visit_expr(expr)) From ff63e805cdfb9a89a9ab814b586c31af83e1ce23 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 19 Oct 2023 23:34:56 +0800 Subject: [PATCH 07/16] build: remove `workspace-hack` from proc-macro crates' dependencies (#12961) --- Cargo.lock | 2 -- src/common/proc_macro/Cargo.toml | 2 -- src/prost/helpers/Cargo.toml | 3 --- 3 files changed, 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb58e6cac3fa1..68a19fffa8f47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6351,7 +6351,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.37", - "workspace-hack", ] [[package]] @@ -7125,7 +7124,6 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "workspace-hack", ] [[package]] diff --git a/src/common/proc_macro/Cargo.toml b/src/common/proc_macro/Cargo.toml index b129cedc0e183..0f86bff6a19d2 100644 --- a/src/common/proc_macro/Cargo.toml +++ b/src/common/proc_macro/Cargo.toml @@ -23,7 +23,5 @@ proc-macro2 = { version = "1", default-features = false } syn = "1" bae = "0.1.7" -[target.'cfg(not(madsim))'.dependencies] -workspace-hack = { path = "../../workspace-hack" } [lints] workspace = true diff --git a/src/prost/helpers/Cargo.toml b/src/prost/helpers/Cargo.toml index 50d9b4febd80b..c78ac3f2a8ece 100644 --- a/src/prost/helpers/Cargo.toml +++ b/src/prost/helpers/Cargo.toml @@ -11,9 +11,6 @@ proc-macro2 = { version = "1", default-features = false } quote = "1" syn = "2" -[target.'cfg(not(madsim))'.dependencies] -workspace-hack = { path = "../../workspace-hack" } - [package.metadata.cargo-machete] ignored = ["workspace-hack"] From d7e8c7e468dd1bea239ec55023bdf2ef306acc1a Mon Sep 17 00:00:00 2001 From: August Date: Fri, 20 Oct 2023 10:32:40 +0800 Subject: [PATCH 08/16] feat: support for all non-streaming related interfaces in catalog controller (#12685) --- src/meta/src/controller/catalog.rs | 688 ++++++++++++++---- src/meta/src/controller/cluster.rs | 63 +- src/meta/src/controller/mod.rs | 21 +- src/meta/src/controller/utils.rs | 286 +++++++- src/meta/src/model_v2/connection.rs | 59 +- src/meta/src/model_v2/database.rs | 68 +- src/meta/src/model_v2/function.rs | 81 ++- src/meta/src/model_v2/index.rs | 38 +- .../migration/src/m20230908_072257_init.rs | 249 ++----- src/meta/src/model_v2/mod.rs | 31 + src/meta/src/model_v2/object.rs | 44 +- src/meta/src/model_v2/object_dependency.rs | 6 +- src/meta/src/model_v2/schema.rs | 75 +- src/meta/src/model_v2/sink.rs | 36 +- src/meta/src/model_v2/source.rs | 38 +- src/meta/src/model_v2/table.rs | 81 +-- src/meta/src/model_v2/user.rs | 4 +- src/meta/src/model_v2/user_privilege.rs | 8 +- src/meta/src/model_v2/view.rs | 52 +- src/meta/src/model_v2/worker.rs | 7 +- src/meta/src/model_v2/worker_property.rs | 4 +- src/meta/src/rpc/ddl_controller.rs | 1 + 22 files changed, 1163 insertions(+), 777 deletions(-) diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 2b992bab70f55..c0cfcf3baba59 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -16,25 +16,35 @@ use std::iter; use itertools::Itertools; use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; -use risingwave_pb::catalog::{PbDatabase, PbSchema}; +use risingwave_pb::catalog::{ + PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, +}; +use risingwave_pb::meta::relation::{PbRelationInfo, RelationInfo}; use risingwave_pb::meta::subscribe_response::{ Info as NotificationInfo, Operation as NotificationOperation, }; -use sea_orm::sea_query::{Expr, Query, QueryStatementBuilder, UnionType}; +use risingwave_pb::meta::{PbRelation, PbRelationGroup}; use sea_orm::{ - ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, DatabaseConnection, - DatabaseTransaction, EntityTrait, ModelTrait, QueryFilter, Statement, TransactionTrait, + ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, DatabaseTransaction, + EntityTrait, QueryFilter, QuerySelect, TransactionTrait, }; use tokio::sync::RwLock; -use crate::controller::utils::construct_obj_dependency_query; +use crate::controller::utils::{ + check_connection_name_duplicate, check_function_signature_duplicate, + check_relation_name_duplicate, check_schema_name_duplicate, ensure_object_id, + ensure_object_not_refer, ensure_schema_empty, ensure_user_id, list_used_by, PartialObject, +}; use crate::controller::ObjectModel; -use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion, UserId}; +use crate::manager::{MetaSrvEnv, NotificationVersion}; +use crate::model_v2::connection::PrivateLinkService; use crate::model_v2::object::ObjectType; use crate::model_v2::prelude::*; use crate::model_v2::{ - connection, database, function, index, object, schema, sink, source, table, view, + connection, database, function, index, object, object_dependency, schema, table, view, + ConnectionId, DatabaseId, FunctionId, ObjectId, SchemaId, SourceId, TableId, UserId, }; +use crate::rpc::ddl_controller::DropMode; use crate::{MetaError, MetaResult}; /// `CatalogController` is the controller for catalog related operations, including database, schema, table, view, etc. @@ -43,10 +53,11 @@ pub struct CatalogController { inner: RwLock, } +#[derive(Clone, Default)] pub struct ReleaseContext { - streaming_jobs: Vec, - source_ids: Vec, - connections: Vec, + streaming_jobs: Vec, + source_ids: Vec, + connections: Vec, } impl CatalogController { @@ -78,6 +89,17 @@ impl CatalogController { .notify_frontend(operation, info) .await } + + async fn notify_frontend_relation_info( + &self, + operation: NotificationOperation, + relation_info: PbRelationInfo, + ) -> NotificationVersion { + self.env + .notification_manager() + .notify_frontend_relation_info(operation, relation_info) + .await + } } impl CatalogController { @@ -89,11 +111,15 @@ impl CatalogController { txn: &DatabaseTransaction, obj_type: ObjectType, owner_id: UserId, + database_id: Option, + schema_id: Option, ) -> MetaResult { let active_db = object::ActiveModel { oid: Default::default(), obj_type: ActiveValue::Set(obj_type), - owner_id: ActiveValue::Set(owner_id as _), + owner_id: ActiveValue::Set(owner_id), + schema_id: ActiveValue::Set(schema_id), + database_id: ActiveValue::Set(database_id), initialized_at: Default::default(), created_at: Default::default(), }; @@ -102,21 +128,23 @@ impl CatalogController { pub async fn create_database(&self, db: PbDatabase) -> MetaResult { let inner = self.inner.write().await; - let txn = inner.db.begin().await?; let owner_id = db.owner; + let txn = inner.db.begin().await?; + ensure_user_id(owner_id, &txn).await?; - let db_obj = Self::create_object(&txn, ObjectType::Database, owner_id).await?; + let db_obj = Self::create_object(&txn, ObjectType::Database, owner_id, None, None).await?; let mut db: database::ActiveModel = db.into(); db.database_id = ActiveValue::Set(db_obj.oid); let db = db.insert(&txn).await?; let mut schemas = vec![]; for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { - let schema_obj = Self::create_object(&txn, ObjectType::Schema, owner_id).await?; + let schema_obj = + Self::create_object(&txn, ObjectType::Schema, owner_id, Some(db_obj.oid), None) + .await?; let schema = schema::ActiveModel { schema_id: ActiveValue::Set(schema_obj.oid), name: ActiveValue::Set(schema_name.into()), - database_id: ActiveValue::Set(db.database_id), }; let schema = schema.insert(&txn).await?; schemas.push(ObjectModel(schema, schema_obj).into()); @@ -138,133 +166,62 @@ impl CatalogController { Ok(version) } - /// List all objects that are using the given one. It runs a recursive CTE to find all the dependencies. - async fn list_used_by(&self, obj_id: i32) -> MetaResult> { - let inner = self.inner.read().await; - - let query = construct_obj_dependency_query(obj_id, "used_by"); - let (sql, values) = query.build_any(&*inner.db.get_database_backend().get_query_builder()); - let res = inner - .db - .query_all(Statement::from_sql_and_values( - inner.db.get_database_backend(), - sql, - values, - )) - .await?; - - let ids: Vec = res - .into_iter() - .map(|row| row.try_get("", "user_by").unwrap()) - .collect_vec(); - Ok(ids) - } - pub async fn drop_database( &self, database_id: DatabaseId, ) -> MetaResult<(ReleaseContext, NotificationVersion)> { let inner = self.inner.write().await; let txn = inner.db.begin().await?; + ensure_object_id(ObjectType::Database, database_id, &txn).await?; - let db = Database::find_by_id(database_id as i32) - .one(&txn) - .await? - .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))?; - let db_obj = db.find_related(Object).one(&txn).await?; - let pb_db = ObjectModel(db.clone(), db_obj.unwrap()).into(); - - let tables = db.find_related(Table).all(&txn).await?; - let sinks = db.find_related(Sink).all(&txn).await?; - let indexes = db.find_related(Index).all(&txn).await?; - let streaming_jobs = tables - .into_iter() - .map(|t| t.table_id) - .chain(sinks.into_iter().map(|s| s.sink_id)) - .chain(indexes.into_iter().map(|i| i.index_id)) - .collect::>(); - let source_ids = db - .find_related(Source) + let streaming_jobs: Vec = Object::find() + .select_only() + .column(object::Column::Oid) + .filter( + object::Column::DatabaseId + .eq(Some(database_id)) + .and(object::Column::ObjType.is_in([ObjectType::Table, ObjectType::Sink])), + ) + .into_tuple() .all(&txn) - .await? - .into_iter() - .map(|s| s.source_id) - .collect_vec(); - let connections = db - .find_related(Connection) + .await?; + + let source_ids: Vec = Object::find() + .select_only() + .column(object::Column::Oid) + .filter( + object::Column::DatabaseId + .eq(Some(database_id)) + .and(object::Column::ObjType.eq(ObjectType::Source)), + ) + .into_tuple() + .all(&txn) + .await?; + + let connections = Connection::find() + .inner_join(Object) + .filter(object::Column::DatabaseId.eq(Some(database_id))) .all(&txn) .await? .into_iter() - .flat_map(|c| c.info) + .map(|conn| conn.info) .collect_vec(); - let mut query_schema = Query::select() - .column(schema::Column::SchemaId) - .from(Schema) - .and_where(schema::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_table = Query::select() - .column(table::Column::TableId) - .from(Table) - .and_where(table::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_source = Query::select() - .column(source::Column::SourceId) - .from(Source) - .and_where(source::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_sink = Query::select() - .column(sink::Column::SinkId) - .from(Sink) - .and_where(sink::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_index = Query::select() - .column(index::Column::IndexId) - .from(Index) - .and_where(index::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_view = Query::select() - .column(view::Column::ViewId) - .from(View) - .and_where(view::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_function = Query::select() - .column(function::Column::FunctionId) - .from(Function) - .and_where(function::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_connection = Query::select() - .column(connection::Column::ConnectionId) - .from(Connection) - .and_where(connection::Column::DatabaseId.eq(database_id as i32)) - .to_owned(); - let query_all = query_schema - .union(UnionType::All, query_table) - .union(UnionType::All, query_source) - .union(UnionType::All, query_sink) - .union(UnionType::All, query_index) - .union(UnionType::All, query_view) - .union(UnionType::All, query_function) - .union(UnionType::All, query_connection) - .union( - UnionType::All, - Query::select() - .expr(Expr::value(database_id as i32)) - .to_owned(), - ) - .to_owned(); - // drop related objects, the relations will be dropped cascade. - Object::delete_many() - .filter(object::Column::Oid.in_subquery(query_all)) - .exec(&txn) - .await?; + // The schema and objects in the database will be delete cascade. + let res = Object::delete_by_id(database_id).exec(&txn).await?; + if res.rows_affected == 0 { + return Err(MetaError::catalog_id_not_found("database", database_id)); + } txn.commit().await?; let version = self .notify_frontend( NotificationOperation::Delete, - NotificationInfo::Database(pb_db), + NotificationInfo::Database(PbDatabase { + id: database_id, + ..Default::default() + }), ) .await; Ok(( @@ -279,11 +236,20 @@ impl CatalogController { pub async fn create_schema(&self, schema: PbSchema) -> MetaResult { let inner = self.inner.write().await; - let txn = inner.db.begin().await?; let owner_id = schema.owner; + let txn = inner.db.begin().await?; + ensure_user_id(owner_id, &txn).await?; + ensure_object_id(ObjectType::Database, schema.database_id, &txn).await?; + check_schema_name_duplicate(&schema.name, schema.database_id, &txn).await?; - // todo: whether to check existence of database and user, or let the database do it? - let schema_obj = Self::create_object(&txn, ObjectType::Schema, owner_id).await?; + let schema_obj = Self::create_object( + &txn, + ObjectType::Schema, + owner_id, + Some(schema.database_id), + None, + ) + .await?; let mut schema: schema::ActiveModel = schema.into(); schema.schema_id = ActiveValue::Set(schema_obj.oid); let schema = schema.insert(&txn).await?; @@ -297,30 +263,486 @@ impl CatalogController { .await; Ok(version) } + + pub async fn drop_schema( + &self, + schema_id: SchemaId, + drop_mode: DropMode, + ) -> MetaResult { + let inner = self.inner.write().await; + let schema_obj = Object::find_by_id(schema_id) + .one(&inner.db) + .await? + .ok_or_else(|| MetaError::catalog_id_not_found("schema", schema_id))?; + if drop_mode == DropMode::Restrict { + ensure_schema_empty(schema_id, &inner.db).await?; + } + + let res = Object::delete(object::ActiveModel { + oid: ActiveValue::Set(schema_id), + ..Default::default() + }) + .exec(&inner.db) + .await?; + if res.rows_affected == 0 { + return Err(MetaError::catalog_id_not_found("schema", schema_id)); + } + + // todo: update user privileges accordingly. + let version = self + .notify_frontend( + NotificationOperation::Delete, + NotificationInfo::Schema(PbSchema { + id: schema_id, + database_id: schema_obj.database_id.unwrap(), + ..Default::default() + }), + ) + .await; + Ok(version) + } + + pub async fn create_function( + &self, + mut pb_function: PbFunction, + ) -> MetaResult { + let inner = self.inner.write().await; + let owner_id = pb_function.owner; + let txn = inner.db.begin().await?; + ensure_user_id(owner_id, &txn).await?; + ensure_object_id(ObjectType::Database, pb_function.database_id, &txn).await?; + ensure_object_id(ObjectType::Schema, pb_function.schema_id, &txn).await?; + check_function_signature_duplicate(&pb_function, &txn).await?; + + let function_obj = Self::create_object( + &txn, + ObjectType::Function, + owner_id, + Some(pb_function.database_id), + Some(pb_function.schema_id), + ) + .await?; + pb_function.id = function_obj.oid; + let function: function::ActiveModel = pb_function.clone().into(); + function.insert(&txn).await?; + txn.commit().await?; + + let version = self + .notify_frontend( + NotificationOperation::Add, + NotificationInfo::Function(pb_function), + ) + .await; + Ok(version) + } + + pub async fn drop_function(&self, function_id: FunctionId) -> MetaResult { + let inner = self.inner.write().await; + let function_obj = Object::find_by_id(function_id) + .one(&inner.db) + .await? + .ok_or_else(|| MetaError::catalog_id_not_found("function", function_id))?; + ensure_object_not_refer(ObjectType::Function, function_id, &inner.db).await?; + + let res = Object::delete_by_id(function_id).exec(&inner.db).await?; + if res.rows_affected == 0 { + return Err(MetaError::catalog_id_not_found("function", function_id)); + } + + let version = self + .notify_frontend( + NotificationOperation::Delete, + NotificationInfo::Function(PbFunction { + id: function_id, + schema_id: function_obj.schema_id.unwrap(), + database_id: function_obj.database_id.unwrap(), + ..Default::default() + }), + ) + .await; + Ok(version) + } + + pub async fn create_connection( + &self, + mut pb_connection: PbConnection, + ) -> MetaResult { + let inner = self.inner.write().await; + let owner_id = pb_connection.owner; + let txn = inner.db.begin().await?; + ensure_user_id(owner_id, &txn).await?; + ensure_object_id(ObjectType::Database, pb_connection.database_id, &txn).await?; + ensure_object_id(ObjectType::Schema, pb_connection.schema_id, &txn).await?; + check_connection_name_duplicate(&pb_connection, &txn).await?; + + let conn_obj = Self::create_object( + &txn, + ObjectType::Connection, + owner_id, + Some(pb_connection.database_id), + Some(pb_connection.schema_id), + ) + .await?; + pb_connection.id = conn_obj.oid; + let connection: connection::ActiveModel = pb_connection.clone().into(); + connection.insert(&txn).await?; + + txn.commit().await?; + + let version = self + .notify_frontend( + NotificationOperation::Add, + NotificationInfo::Connection(pb_connection), + ) + .await; + Ok(version) + } + + pub async fn get_connection_by_id( + &self, + connection_id: ConnectionId, + ) -> MetaResult { + let inner = self.inner.read().await; + let (conn, obj) = Connection::find_by_id(connection_id) + .find_also_related(Object) + .one(&inner.db) + .await? + .ok_or_else(|| MetaError::catalog_id_not_found("connection", connection_id))?; + + Ok(ObjectModel(conn, obj.unwrap()).into()) + } + + pub async fn drop_connection( + &self, + connection_id: ConnectionId, + ) -> MetaResult { + let inner = self.inner.write().await; + let connection_obj = Object::find_by_id(connection_id) + .one(&inner.db) + .await? + .ok_or_else(|| MetaError::catalog_id_not_found("connection", connection_id))?; + ensure_object_not_refer(ObjectType::Connection, connection_id, &inner.db).await?; + + let res = Object::delete_by_id(connection_id).exec(&inner.db).await?; + if res.rows_affected == 0 { + return Err(MetaError::catalog_id_not_found("connection", connection_id)); + } + + let version = self + .notify_frontend( + NotificationOperation::Delete, + NotificationInfo::Connection(PbConnection { + id: connection_id, + schema_id: connection_obj.schema_id.unwrap(), + database_id: connection_obj.database_id.unwrap(), + ..Default::default() + }), + ) + .await; + Ok(version) + } + + pub async fn create_view(&self, mut pb_view: PbView) -> MetaResult { + let inner = self.inner.write().await; + let owner_id = pb_view.owner; + let txn = inner.db.begin().await?; + ensure_user_id(owner_id, &txn).await?; + ensure_object_id(ObjectType::Database, pb_view.database_id, &txn).await?; + ensure_object_id(ObjectType::Schema, pb_view.schema_id, &txn).await?; + check_relation_name_duplicate(&pb_view.name, pb_view.database_id, pb_view.schema_id, &txn) + .await?; + + let view_obj = Self::create_object( + &txn, + ObjectType::View, + owner_id, + Some(pb_view.database_id), + Some(pb_view.schema_id), + ) + .await?; + pb_view.id = view_obj.oid; + let view: view::ActiveModel = pb_view.clone().into(); + view.insert(&txn).await?; + + // todo: change `dependent_relations` to `dependent_objects`, which should includes connection and function as well. + // todo: shall we need to check existence of them Or let database handle it by FOREIGN KEY constraint. + for obj_id in &pb_view.dependent_relations { + object_dependency::ActiveModel { + oid: ActiveValue::Set(*obj_id), + used_by: ActiveValue::Set(view_obj.oid), + ..Default::default() + } + .insert(&txn) + .await?; + } + + txn.commit().await?; + + let version = self + .notify_frontend_relation_info( + NotificationOperation::Add, + PbRelationInfo::View(pb_view), + ) + .await; + Ok(version) + } + + pub async fn drop_relation( + &self, + object_type: ObjectType, + object_id: ObjectId, + drop_mode: DropMode, + ) -> MetaResult<(ReleaseContext, NotificationVersion)> { + let inner = self.inner.write().await; + let txn = inner.db.begin().await?; + let obj: PartialObject = Object::find_by_id(object_id) + .into_partial_model() + .one(&txn) + .await? + .ok_or_else(|| MetaError::catalog_id_not_found(object_type.as_str(), object_id))?; + assert_eq!(obj.obj_type, object_type); + + let mut to_drop_objects = match drop_mode { + DropMode::Cascade => list_used_by(object_id, &txn).await?, + DropMode::Restrict => { + ensure_object_not_refer(object_type, object_id, &txn).await?; + vec![] + } + }; + assert!( + to_drop_objects.iter().all(|obj| matches!( + obj.obj_type, + ObjectType::Table | ObjectType::Index | ObjectType::Sink | ObjectType::View + )), + "only these objects will depends on others" + ); + to_drop_objects.push(obj); + + let to_drop_table_ids = to_drop_objects + .iter() + .filter(|obj| obj.obj_type == ObjectType::Table) + .map(|obj| obj.oid); + let mut to_drop_streaming_jobs = to_drop_objects + .iter() + .filter(|obj| obj.obj_type == ObjectType::Table || obj.obj_type == ObjectType::Sink) + .map(|obj| obj.oid) + .collect_vec(); + // todo: record index dependency info in the object dependency table. + let to_drop_index_ids = to_drop_objects + .iter() + .filter(|obj| obj.obj_type == ObjectType::Index) + .map(|obj| obj.oid) + .collect_vec(); + + // Add associated sources. + let mut to_drop_source_ids: Vec = Table::find() + .select_only() + .column(table::Column::OptionalAssociatedSourceId) + .filter( + table::Column::TableId + .is_in(to_drop_table_ids) + .and(table::Column::OptionalAssociatedSourceId.is_not_null()), + ) + .into_tuple() + .all(&txn) + .await?; + let to_drop_source_objs: Vec = Object::find() + .filter(object::Column::Oid.is_in(to_drop_source_ids.clone())) + .into_partial_model() + .all(&txn) + .await?; + to_drop_objects.extend(to_drop_source_objs.clone()); + if object_type == ObjectType::Source { + to_drop_source_ids.push(object_id); + } + + // add internal tables. + let index_table_ids: Vec = Index::find() + .select_only() + .column(index::Column::IndexTableId) + .filter(index::Column::IndexId.is_in(to_drop_index_ids)) + .into_tuple() + .all(&txn) + .await?; + to_drop_streaming_jobs.extend(index_table_ids); + let to_drop_internal_table_objs: Vec = Object::find() + .filter(object::Column::Oid.is_in(to_drop_streaming_jobs.clone())) + .into_partial_model() + .all(&txn) + .await?; + to_drop_objects.extend(to_drop_internal_table_objs); + + // delete all in to_drop_objects. + let res = Object::delete_many() + .filter(object::Column::Oid.is_in(to_drop_objects.iter().map(|obj| obj.oid))) + .exec(&txn) + .await?; + if res.rows_affected == 0 { + return Err(MetaError::catalog_id_not_found( + object_type.as_str(), + object_id, + )); + } + + // notify about them. + let relations = to_drop_objects + .into_iter() + .map(|obj| match obj.obj_type { + ObjectType::Table => PbRelation { + relation_info: Some(RelationInfo::Table(PbTable { + id: obj.oid, + schema_id: obj.schema_id.unwrap(), + database_id: obj.database_id.unwrap(), + ..Default::default() + })), + }, + ObjectType::Source => PbRelation { + relation_info: Some(RelationInfo::Source(PbSource { + id: obj.oid, + schema_id: obj.schema_id.unwrap(), + database_id: obj.database_id.unwrap(), + ..Default::default() + })), + }, + ObjectType::Sink => PbRelation { + relation_info: Some(RelationInfo::Sink(PbSink { + id: obj.oid, + schema_id: obj.schema_id.unwrap(), + database_id: obj.database_id.unwrap(), + ..Default::default() + })), + }, + ObjectType::View => PbRelation { + relation_info: Some(RelationInfo::View(PbView { + id: obj.oid, + schema_id: obj.schema_id.unwrap(), + database_id: obj.database_id.unwrap(), + ..Default::default() + })), + }, + ObjectType::Index => PbRelation { + relation_info: Some(RelationInfo::Index(PbIndex { + id: obj.oid, + schema_id: obj.schema_id.unwrap(), + database_id: obj.database_id.unwrap(), + ..Default::default() + })), + }, + _ => unreachable!("only relations will be dropped."), + }) + .collect_vec(); + let version = self + .notify_frontend( + NotificationOperation::Delete, + NotificationInfo::RelationGroup(PbRelationGroup { relations }), + ) + .await; + + Ok(( + ReleaseContext { + streaming_jobs: to_drop_streaming_jobs, + source_ids: to_drop_source_ids, + connections: vec![], + }, + version, + )) + } } #[cfg(test)] +#[cfg(not(madsim))] mod tests { use risingwave_common::catalog::DEFAULT_SUPER_USER_ID; use super::*; + const TEST_DATABASE_ID: DatabaseId = 1; + const TEST_SCHEMA_ID: SchemaId = 2; + const TEST_OWNER_ID: UserId = 1; + #[tokio::test] - #[cfg(not(madsim))] - async fn test_create_database() { - let mgr = CatalogController::new(MetaSrvEnv::for_test().await).unwrap(); + async fn test_create_database() -> MetaResult<()> { + let mgr = CatalogController::new(MetaSrvEnv::for_test().await)?; let db = PbDatabase { name: "test".to_string(), owner: DEFAULT_SUPER_USER_ID, ..Default::default() }; - mgr.create_database(db).await.unwrap(); + mgr.create_database(db).await?; + let db = Database::find() .filter(database::Column::Name.eq("test")) .one(&mgr.inner.read().await.db) - .await - .unwrap() + .await? .unwrap(); - mgr.drop_database(db.database_id as _).await.unwrap(); + mgr.drop_database(db.database_id).await?; + Ok(()) + } + + #[tokio::test] + async fn test_create_view() -> MetaResult<()> { + let mgr = CatalogController::new(MetaSrvEnv::for_test().await)?; + let pb_view = PbView { + schema_id: TEST_SCHEMA_ID, + database_id: TEST_DATABASE_ID, + name: "view".to_string(), + owner: TEST_OWNER_ID, + sql: "CREATE VIEW view AS SELECT 1".to_string(), + ..Default::default() + }; + mgr.create_view(pb_view.clone()).await?; + assert!(mgr.create_view(pb_view).await.is_err()); + + let view = View::find().one(&mgr.inner.read().await.db).await?.unwrap(); + mgr.drop_relation(ObjectType::View, view.view_id, DropMode::Cascade) + .await?; + + Ok(()) + } + + #[tokio::test] + async fn test_create_function() -> MetaResult<()> { + let mgr = CatalogController::new(MetaSrvEnv::for_test().await)?; + let return_type = risingwave_pb::data::DataType { + type_name: risingwave_pb::data::data_type::TypeName::Int32 as _, + ..Default::default() + }; + let pb_function = PbFunction { + schema_id: TEST_SCHEMA_ID, + database_id: TEST_DATABASE_ID, + name: "test_function".to_string(), + owner: TEST_OWNER_ID, + arg_types: vec![], + return_type: Some(return_type.clone()), + language: "python".to_string(), + kind: Some(risingwave_pb::catalog::function::Kind::Scalar( + Default::default(), + )), + ..Default::default() + }; + mgr.create_function(pb_function.clone()).await?; + assert!(mgr.create_function(pb_function).await.is_err()); + + let function = Function::find() + .inner_join(Object) + .filter( + object::Column::DatabaseId + .eq(TEST_DATABASE_ID) + .and(object::Column::SchemaId.eq(TEST_SCHEMA_ID)) + .add(function::Column::Name.eq("test_function")), + ) + .one(&mgr.inner.read().await.db) + .await? + .unwrap(); + assert_eq!(function.return_type.0, return_type); + assert_eq!(function.language, "python"); + + mgr.drop_function(function.function_id).await?; + assert!(Object::find_by_id(function.function_id) + .one(&mgr.inner.read().await.db) + .await? + .is_none()); + + Ok(()) } } diff --git a/src/meta/src/controller/cluster.rs b/src/meta/src/controller/cluster.rs index ef45ab1deddef..ca29380a49fca 100644 --- a/src/meta/src/controller/cluster.rs +++ b/src/meta/src/controller/cluster.rs @@ -40,9 +40,9 @@ use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::task::JoinHandle; use crate::manager::prelude::{Worker, WorkerProperty}; -use crate::manager::{LocalNotification, MetaSrvEnv, WorkerId, WorkerKey}; +use crate::manager::{LocalNotification, MetaSrvEnv, WorkerKey}; use crate::model_v2::worker::{WorkerStatus, WorkerType}; -use crate::model_v2::{worker, worker_property, I32Array}; +use crate::model_v2::{worker, worker_property, I32Array, TransactionId, WorkerId}; use crate::{MetaError, MetaResult}; pub type ClusterControllerRef = Arc; @@ -58,7 +58,7 @@ struct WorkerInfo(worker::Model, Option); impl From for PbWorkerNode { fn from(info: WorkerInfo) -> Self { Self { - id: info.0.worker_id as _, + id: info.0.worker_id, r#type: PbWorkerType::from(info.0.worker_type) as _, host: Some(PbHostAddress { host: info.0.host, @@ -74,7 +74,7 @@ impl From for PbWorkerNode { .iter() .map(|&id| PbParallelUnit { id: id as _, - worker_node_id: info.0.worker_id as _, + worker_node_id: info.0.worker_id, }) .collect_vec() }) @@ -84,7 +84,7 @@ impl From for PbWorkerNode { is_serving: p.is_serving, is_unschedulable: p.is_unschedulable, }), - transactional_id: info.0.transaction_id.map(|txn| txn as _), + transactional_id: info.0.transaction_id, } } } @@ -137,7 +137,7 @@ impl From<&PbWorkerNode> for worker::ActiveModel { fn from(worker: &PbWorkerNode) -> Self { let host = worker.host.clone().unwrap(); Self { - worker_id: ActiveValue::Set(worker.id as i32), + worker_id: ActiveValue::Set(worker.id), worker_type: ActiveValue::Set(worker.r#type().into()), host: ActiveValue::Set(host.host), port: ActiveValue::Set(host.port), @@ -312,14 +312,7 @@ impl ClusterController { }; if let Err(err) = Worker::delete_many() - .filter( - worker::Column::WorkerId.is_in( - worker_to_delete - .into_iter() - .map(|id| id as i32) - .collect_vec(), - ), - ) + .filter(worker::Column::WorkerId.is_in(worker_to_delete)) .exec(&inner.db) .await { @@ -457,7 +450,7 @@ pub struct StreamingClusterInfo { pub struct ClusterControllerInner { db: DatabaseConnection, /// Record for tracking available machine ids, one is available. - available_transactional_ids: VecDeque, + available_transactional_ids: VecDeque, worker_extra_info: HashMap, } @@ -466,7 +459,7 @@ impl ClusterControllerInner { pub const MAX_WORKER_REUSABLE_ID_COUNT: usize = 1 << Self::MAX_WORKER_REUSABLE_ID_BITS; pub async fn new(db: DatabaseConnection) -> MetaResult { - let workers: Vec<(i32, Option)> = Worker::find() + let workers: Vec<(WorkerId, Option)> = Worker::find() .select_only() .column(worker::Column::WorkerId) .column(worker::Column::TransactionId) @@ -478,13 +471,13 @@ impl ClusterControllerInner { .cloned() .filter_map(|(_, txn_id)| txn_id) .collect(); - let available_transactional_ids = (0..Self::MAX_WORKER_REUSABLE_ID_COUNT as i32) + let available_transactional_ids = (0..Self::MAX_WORKER_REUSABLE_ID_COUNT as TransactionId) .filter(|id| !inuse_txn_ids.contains(id)) .collect(); let worker_extra_info = workers .into_iter() - .map(|(w, _)| (w as u32, WorkerExtraInfo::default())) + .map(|(w, _)| (w, WorkerExtraInfo::default())) .collect(); Ok(Self { @@ -527,7 +520,7 @@ impl ClusterControllerInner { } } - fn apply_transaction_id(&self, r#type: PbWorkerType) -> MetaResult> { + fn apply_transaction_id(&self, r#type: PbWorkerType) -> MetaResult> { match (self.available_transactional_ids.front(), r#type) { (None, _) => Err(MetaError::unavailable( "no available reusable machine id".to_string(), @@ -548,9 +541,9 @@ impl ClusterControllerInner { let txn = self.db.begin().await?; // TODO: remove this workaround when we deprecate parallel unit ids. - let derive_parallel_units = |txn_id: i32, start: u32, end: u32| { + let derive_parallel_units = |txn_id: TransactionId, start: u32, end: u32| { (start..end) - .map(|idx| (idx << Self::MAX_WORKER_REUSABLE_ID_BITS) as i32 + txn_id) + .map(|idx| ((idx << Self::MAX_WORKER_REUSABLE_ID_BITS) + txn_id) as i32) .collect_vec() }; @@ -607,11 +600,11 @@ impl ClusterControllerInner { WorkerProperty::update(property).exec(&txn).await?; txn.commit().await?; - self.update_worker_ttl(worker.worker_id as _, ttl)?; - Ok(worker.worker_id as WorkerId) + self.update_worker_ttl(worker.worker_id, ttl)?; + Ok(worker.worker_id) } else { - self.update_worker_ttl(worker.worker_id as _, ttl)?; - Ok(worker.worker_id as WorkerId) + self.update_worker_ttl(worker.worker_id, ttl)?; + Ok(worker.worker_id) }; } let txn_id = self.apply_transaction_id(r#type)?; @@ -628,7 +621,7 @@ impl ClusterControllerInner { let worker_id = insert_res.last_insert_id as WorkerId; if r#type == PbWorkerType::ComputeNode { let property = worker_property::ActiveModel { - worker_id: ActiveValue::Set(insert_res.last_insert_id), + worker_id: ActiveValue::Set(worker_id), parallel_unit_ids: ActiveValue::Set(I32Array(derive_parallel_units( *txn_id.as_ref().unwrap(), 0, @@ -653,7 +646,7 @@ impl ClusterControllerInner { pub async fn activate_worker(&self, worker_id: WorkerId) -> MetaResult { let worker = worker::ActiveModel { - worker_id: ActiveValue::Set(worker_id as i32), + worker_id: ActiveValue::Set(worker_id), status: ActiveValue::Set(WorkerStatus::Running), ..Default::default() }; @@ -676,10 +669,7 @@ impl ClusterControllerInner { worker_property::Column::IsUnschedulable, Expr::value(is_unschedulable), ) - .filter( - worker_property::Column::WorkerId - .is_in(worker_ids.into_iter().map(|id| id as i32).collect_vec()), - ) + .filter(worker_property::Column::WorkerId.is_in(worker_ids)) .exec(&self.db) .await?; @@ -707,10 +697,9 @@ impl ClusterControllerInner { return Err(MetaError::invalid_parameter("worker not found!")); } - self.worker_extra_info - .remove(&(worker.worker_id as WorkerId)); + self.worker_extra_info.remove(&worker.worker_id); if let Some(txn_id) = &worker.transaction_id { - self.available_transactional_ids.push_back(*txn_id as _); + self.available_transactional_ids.push_back(*txn_id); } Ok(WorkerInfo(worker, property).into()) } @@ -776,7 +765,7 @@ impl ClusterControllerInner { } pub async fn list_active_parallel_units(&self) -> MetaResult> { - let parallel_units: Vec<(i32, I32Array)> = WorkerProperty::find() + let parallel_units: Vec<(WorkerId, I32Array)> = WorkerProperty::find() .select_only() .column(worker_property::Column::WorkerId) .column(worker_property::Column::ParallelUnitIds) @@ -790,7 +779,7 @@ impl ClusterControllerInner { .flat_map(|(id, pu)| { pu.0.into_iter().map(move |parallel_unit_id| ParallelUnit { id: parallel_unit_id as _, - worker_node_id: id as _, + worker_node_id: id, }) }) .collect_vec()) @@ -848,7 +837,7 @@ impl ClusterControllerInner { } pub async fn get_worker_by_id(&self, worker_id: WorkerId) -> MetaResult> { - let worker = Worker::find_by_id(worker_id as i32) + let worker = Worker::find_by_id(worker_id) .find_also_related(WorkerProperty) .one(&self.db) .await?; diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index 3fcb6a7d004ae..5b0ff4ab99bef 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -13,10 +13,11 @@ // limitations under the License. use anyhow::anyhow; -use risingwave_pb::catalog::{PbDatabase, PbSchema}; +use risingwave_pb::catalog::connection::PbInfo as PbConnectionInfo; +use risingwave_pb::catalog::{PbConnection, PbDatabase, PbSchema}; use sea_orm::{ActiveValue, DatabaseConnection, ModelTrait}; -use crate::model_v2::{database, object, schema}; +use crate::model_v2::{connection, database, object, schema}; use crate::MetaError; #[allow(dead_code)] @@ -81,7 +82,6 @@ impl From for schema::ActiveModel { Self { schema_id: ActiveValue::Set(schema.id as _), name: ActiveValue::Set(schema.name), - database_id: ActiveValue::Set(schema.database_id as _), } } } @@ -91,8 +91,21 @@ impl From> for PbSchema { Self { id: value.0.schema_id as _, name: value.0.name, - database_id: value.0.database_id as _, + database_id: value.1.database_id.unwrap() as _, owner: value.1.owner_id as _, } } } + +impl From> for PbConnection { + fn from(value: ObjectModel) -> Self { + Self { + id: value.1.oid as _, + schema_id: value.1.schema_id.unwrap() as _, + database_id: value.1.database_id.unwrap() as _, + name: value.0.name, + owner: value.1.owner_id as _, + info: Some(PbConnectionInfo::PrivateLinkService(value.0.info.0)), + } + } +} diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs index 9e2e75f9eb10b..964ee24ae99b5 100644 --- a/src/meta/src/controller/utils.rs +++ b/src/meta/src/controller/utils.rs @@ -12,16 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. +use anyhow::anyhow; use model_migration::WithQuery; +use risingwave_pb::catalog::{PbConnection, PbFunction}; use sea_orm::sea_query::{ - Alias, CommonTableExpression, Expr, Query, SelectStatement, UnionType, WithClause, + Alias, CommonTableExpression, Expr, Query, QueryStatementBuilder, SelectStatement, UnionType, + WithClause, +}; +use sea_orm::{ + ColumnTrait, ConnectionTrait, DerivePartialModel, EntityTrait, FromQueryResult, JoinType, + Order, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait, Statement, }; -use sea_orm::{ColumnTrait, JoinType, Order}; -use crate::model_v2::object_dependency; +use crate::model_v2::object::ObjectType; use crate::model_v2::prelude::*; +use crate::model_v2::{ + connection, function, index, object, object_dependency, schema, sink, source, table, view, + DataTypeArray, DatabaseId, ObjectId, SchemaId, UserId, +}; +use crate::{MetaError, MetaResult}; -/// This function will construct a query using recursive cte to find all objects that are used by the given object. +/// This function will construct a query using recursive cte to find all objects[(id, `obj_type`)] that are used by the given object. /// /// # Examples /// @@ -30,26 +41,26 @@ use crate::model_v2::prelude::*; /// use sea_orm::sea_query::*; /// use sea_orm::*; /// -/// let query = construct_obj_dependency_query(1, "used_by"); +/// let query = construct_obj_dependency_query(1); /// /// assert_eq!( /// query.to_string(MysqlQueryBuilder), -/// r#"WITH RECURSIVE `used_by_object_ids` (`used_by`) AS (SELECT `used_by` FROM `object_dependency` WHERE `object_dependency`.`oid` = 1 UNION ALL (SELECT `object_dependency`.`used_by` FROM `object_dependency` INNER JOIN `used_by_object_ids` ON `used_by_object_ids`.`used_by` = `oid`)) SELECT DISTINCT `used_by` FROM `used_by_object_ids` ORDER BY `used_by` DESC"# +/// r#"WITH RECURSIVE `used_by_object_ids` (`used_by`) AS (SELECT `used_by` FROM `object_dependency` WHERE `object_dependency`.`oid` = 1 UNION ALL (SELECT `object_dependency`.`used_by` FROM `object_dependency` INNER JOIN `used_by_object_ids` ON `used_by_object_ids`.`used_by` = `oid`)) SELECT DISTINCT `oid`, `obj_type`, `schema_id`, `database_id` FROM `used_by_object_ids` INNER JOIN `object` ON `used_by_object_ids`.`used_by` = `oid` ORDER BY `oid` DESC"# /// ); /// assert_eq!( /// query.to_string(PostgresQueryBuilder), -/// r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL (SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid")) SELECT DISTINCT "used_by" FROM "used_by_object_ids" ORDER BY "used_by" DESC"# +/// r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL (SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid")) SELECT DISTINCT "oid", "obj_type", "schema_id", "database_id" FROM "used_by_object_ids" INNER JOIN "object" ON "used_by_object_ids"."used_by" = "oid" ORDER BY "oid" DESC"# /// ); /// assert_eq!( /// query.to_string(SqliteQueryBuilder), -/// r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid") SELECT DISTINCT "used_by" FROM "used_by_object_ids" ORDER BY "used_by" DESC"# +/// r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid") SELECT DISTINCT "oid", "obj_type", "schema_id", "database_id" FROM "used_by_object_ids" INNER JOIN "object" ON "used_by_object_ids"."used_by" = "oid" ORDER BY "oid" DESC"# /// ); /// ``` -pub fn construct_obj_dependency_query(obj_id: i32, column: &str) -> WithQuery { +pub fn construct_obj_dependency_query(obj_id: ObjectId) -> WithQuery { let cte_alias = Alias::new("used_by_object_ids"); - let cte_return_alias = Alias::new(column); + let cte_return_alias = Alias::new("used_by"); - let base_query = SelectStatement::new() + let mut base_query = SelectStatement::new() .column(object_dependency::Column::UsedBy) .from(ObjectDependency) .and_where(object_dependency::Column::Oid.eq(obj_id)) @@ -58,8 +69,7 @@ pub fn construct_obj_dependency_query(obj_id: i32, column: &str) -> WithQuery { let cte_referencing = Query::select() .column((ObjectDependency, object_dependency::Column::UsedBy)) .from(ObjectDependency) - .join( - JoinType::InnerJoin, + .inner_join( cte_alias.clone(), Expr::col((cte_alias.clone(), cte_return_alias.clone())) .equals(object_dependency::Column::Oid), @@ -67,21 +77,25 @@ pub fn construct_obj_dependency_query(obj_id: i32, column: &str) -> WithQuery { .to_owned(); let common_table_expr = CommonTableExpression::new() - .query( - base_query - .clone() - .union(UnionType::All, cte_referencing) - .to_owned(), - ) + .query(base_query.union(UnionType::All, cte_referencing).to_owned()) .column(cte_return_alias.clone()) .table_name(cte_alias.clone()) .to_owned(); SelectStatement::new() .distinct() - .column(cte_return_alias.clone()) - .from(cte_alias) - .order_by(cte_return_alias.clone(), Order::Desc) + .columns([ + object::Column::Oid, + object::Column::ObjType, + object::Column::SchemaId, + object::Column::DatabaseId, + ]) + .from(cte_alias.clone()) + .inner_join( + Object, + Expr::col((cte_alias, cte_return_alias.clone())).equals(object::Column::Oid), + ) + .order_by(object::Column::Oid, Order::Desc) .to_owned() .with( WithClause::new() @@ -91,3 +105,231 @@ pub fn construct_obj_dependency_query(obj_id: i32, column: &str) -> WithQuery { ) .to_owned() } + +#[derive(Clone, DerivePartialModel, FromQueryResult)] +#[sea_orm(entity = "Object")] +pub struct PartialObject { + pub oid: ObjectId, + pub obj_type: ObjectType, + pub schema_id: Option, + pub database_id: Option, +} + +/// List all objects that are using the given one. It runs a recursive CTE to find all the dependencies. +pub async fn list_used_by(obj_id: ObjectId, db: &C) -> MetaResult> +where + C: ConnectionTrait, +{ + let query = construct_obj_dependency_query(obj_id); + let (sql, values) = query.build_any(&*db.get_database_backend().get_query_builder()); + let objects = PartialObject::find_by_statement(Statement::from_sql_and_values( + db.get_database_backend(), + sql, + values, + )) + .all(db) + .await?; + Ok(objects) +} + +/// `ensure_object_id` ensures the existence of target object in the cluster. +pub async fn ensure_object_id( + object_type: ObjectType, + obj_id: ObjectId, + db: &C, +) -> MetaResult<()> +where + C: ConnectionTrait, +{ + let count = Object::find_by_id(obj_id).count(db).await?; + if count == 0 { + return Err(MetaError::catalog_id_not_found( + object_type.as_str(), + obj_id, + )); + } + Ok(()) +} + +/// `ensure_user_id` ensures the existence of target user in the cluster. +pub async fn ensure_user_id(user_id: UserId, db: &C) -> MetaResult<()> +where + C: ConnectionTrait, +{ + let count = User::find_by_id(user_id).count(db).await?; + if count == 0 { + return Err(anyhow!("user {} was concurrently dropped", user_id).into()); + } + Ok(()) +} + +/// `check_function_signature_duplicate` checks whether the function name and its signature is already used in the target namespace. +pub async fn check_function_signature_duplicate( + pb_function: &PbFunction, + db: &C, +) -> MetaResult<()> +where + C: ConnectionTrait, +{ + let count = Function::find() + .inner_join(Object) + .filter( + object::Column::DatabaseId + .eq(pb_function.database_id as DatabaseId) + .and(object::Column::SchemaId.eq(pb_function.schema_id as SchemaId)) + .and(function::Column::Name.eq(&pb_function.name)) + .and(function::Column::ArgTypes.eq(DataTypeArray(pb_function.arg_types.clone()))), + ) + .count(db) + .await?; + if count > 0 { + assert_eq!(count, 1); + return Err(MetaError::catalog_duplicated("function", &pb_function.name)); + } + Ok(()) +} + +/// `check_connection_name_duplicate` checks whether the connection name is already used in the target namespace. +pub async fn check_connection_name_duplicate( + pb_connection: &PbConnection, + db: &C, +) -> MetaResult<()> +where + C: ConnectionTrait, +{ + let count = Connection::find() + .inner_join(Object) + .filter( + object::Column::DatabaseId + .eq(pb_connection.database_id as DatabaseId) + .and(object::Column::SchemaId.eq(pb_connection.schema_id as SchemaId)) + .and(connection::Column::Name.eq(&pb_connection.name)), + ) + .count(db) + .await?; + if count > 0 { + assert_eq!(count, 1); + return Err(MetaError::catalog_duplicated( + "connection", + &pb_connection.name, + )); + } + Ok(()) +} + +/// `check_relation_name_duplicate` checks whether the relation name is already used in the target namespace. +pub async fn check_relation_name_duplicate( + name: &str, + database_id: DatabaseId, + schema_id: SchemaId, + db: &C, +) -> MetaResult<()> +where + C: ConnectionTrait, +{ + macro_rules! check_duplicated { + ($obj_type:expr, $entity:ident, $table:ident) => { + let count = Object::find() + .inner_join($entity) + .filter( + object::Column::DatabaseId + .eq(Some(database_id)) + .and(object::Column::SchemaId.eq(Some(schema_id))) + .and($table::Column::Name.eq(name)), + ) + .count(db) + .await?; + if count != 0 { + return Err(MetaError::catalog_duplicated($obj_type.as_str(), name)); + } + }; + } + check_duplicated!(ObjectType::Table, Table, table); + check_duplicated!(ObjectType::Source, Source, source); + check_duplicated!(ObjectType::Sink, Sink, sink); + check_duplicated!(ObjectType::Index, Index, index); + check_duplicated!(ObjectType::View, View, view); + + Ok(()) +} + +/// `check_schema_name_duplicate` checks whether the schema name is already used in the target database. +pub async fn check_schema_name_duplicate( + name: &str, + database_id: DatabaseId, + db: &C, +) -> MetaResult<()> +where + C: ConnectionTrait, +{ + let count = Object::find() + .inner_join(Schema) + .filter( + object::Column::ObjType + .eq(ObjectType::Schema) + .and(object::Column::DatabaseId.eq(Some(database_id))) + .and(schema::Column::Name.eq(name)), + ) + .count(db) + .await?; + if count != 0 { + return Err(MetaError::catalog_duplicated("schema", name)); + } + + Ok(()) +} + +/// `ensure_object_not_refer` ensures that object are not used by any other ones except indexes. +pub async fn ensure_object_not_refer( + object_type: ObjectType, + object_id: ObjectId, + db: &C, +) -> MetaResult<()> +where + C: ConnectionTrait, +{ + // Ignore indexes. + let count = if object_type == ObjectType::Table { + ObjectDependency::find() + .join( + JoinType::InnerJoin, + object_dependency::Relation::Object1.def(), + ) + .filter( + object_dependency::Column::Oid + .eq(object_id) + .and(object::Column::ObjType.ne(ObjectType::Index)), + ) + .count(db) + .await? + } else { + ObjectDependency::find() + .filter(object_dependency::Column::Oid.eq(object_id)) + .count(db) + .await? + }; + if count != 0 { + return Err(MetaError::permission_denied(format!( + "{} used by {} other objects.", + object_type.as_str(), + count + ))); + } + Ok(()) +} + +/// `ensure_schema_empty` ensures that the schema is empty, used by `DROP SCHEMA`. +pub async fn ensure_schema_empty(schema_id: SchemaId, db: &C) -> MetaResult<()> +where + C: ConnectionTrait, +{ + let count = Object::find() + .filter(object::Column::SchemaId.eq(Some(schema_id))) + .count(db) + .await?; + if count != 0 { + return Err(MetaError::permission_denied("schema is not empty".into())); + } + + Ok(()) +} diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/src/model_v2/connection.rs index 6f56002780411..f6638ed0b53a4 100644 --- a/src/meta/src/model_v2/connection.rs +++ b/src/meta/src/model_v2/connection.rs @@ -12,29 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_pb::catalog::connection::{PbInfo, PbPrivateLinkService}; +use risingwave_pb::catalog::PbConnection; use sea_orm::entity::prelude::*; +use sea_orm::{ActiveValue, FromJsonQueryResult}; +use serde::{Deserialize, Serialize}; + +use crate::model_v2::ConnectionId; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "connection")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub connection_id: i32, + pub connection_id: ConnectionId, pub name: String, - pub schema_id: i32, - pub database_id: i32, - pub info: Option, + pub info: PrivateLinkService, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::ConnectionId", @@ -43,38 +39,18 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, #[sea_orm(has_many = "super::sink::Entity")] Sink, #[sea_orm(has_many = "super::source::Entity")] Source, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Object.def() } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Sink.def() @@ -88,3 +64,22 @@ impl Related for Entity { } impl ActiveModelBehavior for ActiveModel {} + +#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Serialize, Deserialize, Default)] +pub struct PrivateLinkService(pub PbPrivateLinkService); + +impl Eq for PrivateLinkService {} + +impl From for ActiveModel { + fn from(conn: PbConnection) -> Self { + let Some(PbInfo::PrivateLinkService(private_link_srv)) = conn.info else { + unreachable!("private link not provided.") + }; + + Self { + connection_id: ActiveValue::Set(conn.id as _), + name: ActiveValue::Set(conn.name), + info: ActiveValue::Set(PrivateLinkService(private_link_srv)), + } + } +} diff --git a/src/meta/src/model_v2/database.rs b/src/meta/src/model_v2/database.rs index dca87c32b9f44..909c12eceac5a 100644 --- a/src/meta/src/model_v2/database.rs +++ b/src/meta/src/model_v2/database.rs @@ -14,23 +14,19 @@ use sea_orm::entity::prelude::*; +use crate::model_v2::DatabaseId; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "database")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub database_id: i32, + pub database_id: DatabaseId, #[sea_orm(unique)] pub name: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::connection::Entity")] - Connection, - #[sea_orm(has_many = "super::function::Entity")] - Function, - #[sea_orm(has_many = "super::index::Entity")] - Index, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::DatabaseId", @@ -39,34 +35,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm(has_many = "super::schema::Entity")] - Schema, - #[sea_orm(has_many = "super::sink::Entity")] - Sink, - #[sea_orm(has_many = "super::source::Entity")] - Source, - #[sea_orm(has_many = "super::table::Entity")] - Table, - #[sea_orm(has_many = "super::view::Entity")] - View, -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Connection.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Function.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Index.def() - } } impl Related for Entity { @@ -75,34 +43,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Sink.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Source.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Table.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::View.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/function.rs b/src/meta/src/model_v2/function.rs index e60f9ea5a2bde..663f8e2284fd7 100644 --- a/src/meta/src/model_v2/function.rs +++ b/src/meta/src/model_v2/function.rs @@ -12,34 +12,40 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_pb::catalog::function::Kind; +use risingwave_pb::catalog::PbFunction; use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue; + +use crate::model_v2::{DataType, DataTypeArray, FunctionId}; + +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum FunctionKind { + #[sea_orm(string_value = "Scalar")] + Scalar, + #[sea_orm(string_value = "Table")] + Table, + #[sea_orm(string_value = "Aggregate")] + Aggregate, +} #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "function")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub function_id: i32, + pub function_id: FunctionId, pub name: String, - pub schema_id: i32, - pub database_id: i32, - pub arg_types: Option, - pub return_type: Option, - pub language: Option, - pub link: Option, - pub identifier: Option, - pub kind: Option, + pub arg_types: DataTypeArray, + pub return_type: DataType, + pub language: String, + pub link: String, + pub identifier: String, + pub kind: FunctionKind, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::FunctionId", @@ -48,20 +54,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } } impl Related for Entity { @@ -70,10 +62,29 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() +impl ActiveModelBehavior for ActiveModel {} + +impl From for FunctionKind { + fn from(kind: Kind) -> Self { + match kind { + Kind::Scalar(_) => Self::Scalar, + Kind::Table(_) => Self::Table, + Kind::Aggregate(_) => Self::Aggregate, + } } } -impl ActiveModelBehavior for ActiveModel {} +impl From for ActiveModel { + fn from(function: PbFunction) -> Self { + Self { + function_id: ActiveValue::Set(function.id as _), + name: ActiveValue::Set(function.name), + arg_types: ActiveValue::Set(DataTypeArray(function.arg_types)), + return_type: ActiveValue::Set(DataType(function.return_type.unwrap())), + language: ActiveValue::Set(function.language), + link: ActiveValue::Set(function.link), + identifier: ActiveValue::Set(function.identifier), + kind: ActiveValue::Set(function.kind.unwrap().into()), + } + } +} diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs index 4cd3cf212a03c..6a4b7d1b349ca 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/src/model_v2/index.rs @@ -14,32 +14,22 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::I32Array; +use crate::model_v2::{I32Array, IndexId, TableId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "index")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub index_id: i32, + pub index_id: IndexId, pub name: String, - pub schema_id: i32, - pub database_id: i32, - pub index_table_id: i32, - pub primary_table_id: i32, + pub index_table_id: TableId, + pub primary_table_id: TableId, pub index_items: Option, pub original_columns: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::IndexId", @@ -48,14 +38,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, #[sea_orm( belongs_to = "super::table::Entity", from = "Column::IndexTableId", @@ -74,22 +56,10 @@ pub enum Relation { Table1, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Object.def() } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index e2bb65d68a849..35d1ab29020e6 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -148,6 +148,8 @@ impl MigrationTrait for Migration { ) .col(ColumnDef::new(Object::ObjType).string().not_null()) .col(ColumnDef::new(Object::OwnerId).integer().not_null()) + .col(ColumnDef::new(Object::SchemaId).integer()) + .col(ColumnDef::new(Object::DatabaseId).integer()) .col( ColumnDef::new(Object::InitializedAt) .timestamp() @@ -165,6 +167,23 @@ impl MigrationTrait for Migration { .name("FK_object_owner_id") .from(Object::Table, Object::OwnerId) .to(User::Table, User::UserId) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_object_database_id") + .from(Object::Table, Object::DatabaseId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_object_schema_id") + .from(Object::Table, Object::SchemaId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .to_owned(), @@ -281,14 +300,6 @@ impl MigrationTrait for Migration { .table(Schema::Table) .col(ColumnDef::new(Schema::SchemaId).integer().primary_key()) .col(ColumnDef::new(Schema::Name).string().not_null()) - .col(ColumnDef::new(Schema::DatabaseId).integer().not_null()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_schema_database_id") - .from(Schema::Table, Schema::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) .foreign_key( &mut ForeignKey::create() .name("FK_schema_object_id") @@ -377,23 +388,7 @@ impl MigrationTrait for Migration { .primary_key(), ) .col(ColumnDef::new(Connection::Name).string().not_null()) - .col(ColumnDef::new(Connection::SchemaId).integer().not_null()) - .col(ColumnDef::new(Connection::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Connection::Info).json()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_connection_database_id") - .from(Connection::Table, Connection::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_connection_schema_id") - .from(Connection::Table, Connection::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) + .col(ColumnDef::new(Connection::Info).json().not_null()) .foreign_key( &mut ForeignKey::create() .name("FK_connection_object_id") @@ -411,8 +406,6 @@ impl MigrationTrait for Migration { .table(Source::Table) .col(ColumnDef::new(Source::SourceId).integer().primary_key()) .col(ColumnDef::new(Source::Name).string().not_null()) - .col(ColumnDef::new(Source::SchemaId).integer().not_null()) - .col(ColumnDef::new(Source::DatabaseId).integer().not_null()) .col(ColumnDef::new(Source::RowIdIndex).string()) .col(ColumnDef::new(Source::Columns).json()) .col(ColumnDef::new(Source::PkColumnIds).json()) @@ -422,20 +415,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Source::WatermarkDescs).json()) .col(ColumnDef::new(Source::OptionalAssociatedTableId).integer()) .col(ColumnDef::new(Source::ConnectionId).integer()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_source_database_id") - .from(Source::Table, Source::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_source_schema_id") - .from(Source::Table, Source::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) .foreign_key( &mut ForeignKey::create() .name("FK_source_object_id") @@ -460,41 +439,37 @@ impl MigrationTrait for Migration { .table(Table::Table) .col(ColumnDef::new(Table::TableId).integer().primary_key()) .col(ColumnDef::new(Table::Name).string().not_null()) - .col(ColumnDef::new(Table::SchemaId).integer().not_null()) - .col(ColumnDef::new(Table::DatabaseId).integer().not_null()) .col(ColumnDef::new(Table::OptionalAssociatedSourceId).integer()) - .col(ColumnDef::new(Table::TableType).string()) - .col(ColumnDef::new(Table::Columns).json()) - .col(ColumnDef::new(Table::Pk).json()) - .col(ColumnDef::new(Table::DistributionKey).json()) - .col(ColumnDef::new(Table::AppendOnly).boolean()) - .col(ColumnDef::new(Table::Properties).json()) - .col(ColumnDef::new(Table::FragmentId).integer()) + .col(ColumnDef::new(Table::TableType).string().not_null()) + .col(ColumnDef::new(Table::Columns).json().not_null()) + .col(ColumnDef::new(Table::Pk).json().not_null()) + .col(ColumnDef::new(Table::DistributionKey).json().not_null()) + .col(ColumnDef::new(Table::AppendOnly).boolean().not_null()) + .col(ColumnDef::new(Table::Properties).json().not_null()) + .col(ColumnDef::new(Table::FragmentId).integer().not_null()) .col(ColumnDef::new(Table::VnodeColIndex).integer()) - .col(ColumnDef::new(Table::ValueIndices).json()) - .col(ColumnDef::new(Table::Definition).string()) - .col(ColumnDef::new(Table::HandlePkConflictBehavior).integer()) - .col(ColumnDef::new(Table::ReadPrefixLenHint).integer()) - .col(ColumnDef::new(Table::WatermarkIndices).json()) - .col(ColumnDef::new(Table::DistKeyInPk).json()) + .col(ColumnDef::new(Table::ValueIndices).json().not_null()) + .col(ColumnDef::new(Table::Definition).string().not_null()) + .col( + ColumnDef::new(Table::HandlePkConflictBehavior) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(Table::ReadPrefixLenHint) + .integer() + .not_null(), + ) + .col(ColumnDef::new(Table::WatermarkIndices).json().not_null()) + .col(ColumnDef::new(Table::DistKeyInPk).json().not_null()) .col(ColumnDef::new(Table::DmlFragmentId).integer()) .col(ColumnDef::new(Table::Cardinality).json()) - .col(ColumnDef::new(Table::CleanedByWatermark).boolean()) - .col(ColumnDef::new(Table::Version).json()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_table_database_id") - .from(Table::Table, Table::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_table_schema_id") - .from(Table::Table, Table::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), + .col( + ColumnDef::new(Table::CleanedByWatermark) + .boolean() + .not_null(), ) + .col(ColumnDef::new(Table::Version).json().not_null()) .foreign_key( &mut ForeignKey::create() .name("FK_view_object_id") @@ -533,8 +508,6 @@ impl MigrationTrait for Migration { .table(Sink::Table) .col(ColumnDef::new(Sink::SinkId).integer().primary_key()) .col(ColumnDef::new(Sink::Name).string().not_null()) - .col(ColumnDef::new(Sink::SchemaId).integer().not_null()) - .col(ColumnDef::new(Sink::DatabaseId).integer().not_null()) .col(ColumnDef::new(Sink::Columns).json()) .col(ColumnDef::new(Sink::PkColumnIds).json()) .col(ColumnDef::new(Sink::DistributionKey).json()) @@ -545,20 +518,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Sink::ConnectionId).integer()) .col(ColumnDef::new(Sink::DbName).string().not_null()) .col(ColumnDef::new(Sink::SinkFromName).string().not_null()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_sink_database_id") - .from(Sink::Table, Sink::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_sink_schema_id") - .from(Sink::Table, Sink::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) .foreign_key( &mut ForeignKey::create() .name("FK_sink_object_id") @@ -583,25 +542,9 @@ impl MigrationTrait for Migration { .table(View::Table) .col(ColumnDef::new(View::ViewId).integer().primary_key()) .col(ColumnDef::new(View::Name).string().not_null()) - .col(ColumnDef::new(View::SchemaId).integer().not_null()) - .col(ColumnDef::new(View::DatabaseId).integer().not_null()) - .col(ColumnDef::new(View::Properties).json()) - .col(ColumnDef::new(View::Sql).string()) - .col(ColumnDef::new(View::Columns).json()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_view_database_id") - .from(View::Table, View::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_view_schema_id") - .from(View::Table, View::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) + .col(ColumnDef::new(View::Properties).json().not_null()) + .col(ColumnDef::new(View::Sql).string().not_null()) + .col(ColumnDef::new(View::Columns).json().not_null()) .foreign_key( &mut ForeignKey::create() .name("FK_view_object_id") @@ -619,26 +562,10 @@ impl MigrationTrait for Migration { .table(Index::Table) .col(ColumnDef::new(Index::IndexId).integer().primary_key()) .col(ColumnDef::new(Index::Name).string().not_null()) - .col(ColumnDef::new(Index::SchemaId).integer().not_null()) - .col(ColumnDef::new(Index::DatabaseId).integer().not_null()) .col(ColumnDef::new(Index::IndexTableId).integer().not_null()) .col(ColumnDef::new(Index::PrimaryTableId).integer().not_null()) .col(ColumnDef::new(Index::IndexItems).json()) .col(ColumnDef::new(Index::OriginalColumns).json()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_index_database_id") - .from(Index::Table, Index::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_index_schema_id") - .from(Index::Table, Index::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) .foreign_key( &mut ForeignKey::create() .name("FK_index_object_id") @@ -670,28 +597,12 @@ impl MigrationTrait for Migration { .table(Function::Table) .col(ColumnDef::new(Function::FunctionId).integer().primary_key()) .col(ColumnDef::new(Function::Name).string().not_null()) - .col(ColumnDef::new(Function::SchemaId).integer().not_null()) - .col(ColumnDef::new(Function::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Function::ArgTypes).json()) - .col(ColumnDef::new(Function::ReturnType).string()) - .col(ColumnDef::new(Function::Language).string()) - .col(ColumnDef::new(Function::Link).string()) - .col(ColumnDef::new(Function::Identifier).string()) - .col(ColumnDef::new(Function::Kind).json()) - .foreign_key( - &mut ForeignKey::create() - .name("FK_function_database_id") - .from(Function::Table, Function::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .foreign_key( - &mut ForeignKey::create() - .name("FK_function_schema_id") - .from(Function::Table, Function::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) + .col(ColumnDef::new(Function::ArgTypes).json().not_null()) + .col(ColumnDef::new(Function::ReturnType).json().not_null()) + .col(ColumnDef::new(Function::Language).string().not_null()) + .col(ColumnDef::new(Function::Link).string().not_null()) + .col(ColumnDef::new(Function::Identifier).string().not_null()) + .col(ColumnDef::new(Function::Kind).string().not_null()) .foreign_key( &mut ForeignKey::create() .name("FK_function_object_id") @@ -774,17 +685,6 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; - manager - .create_index( - MigrationIndex::create() - .table(Schema::Table) - .name("idx_schema_database_id_name") - .unique() - .col(Schema::DatabaseId) - .col(Schema::Name) - .to_owned(), - ) - .await?; // 4. initialize data. let insert_cluster_id = Query::insert() @@ -820,12 +720,12 @@ impl MigrationTrait for Migration { // Since User table is newly created, we assume that the initial user id of `root` is 1 and `postgres` is 2. let insert_objects = Query::insert() .into_table(Object::Table) - .columns([Object::ObjType, Object::OwnerId]) - .values_panic(["DATABASE".into(), 1.into()]) - .values_panic(["SCHEMA".into(), 1.into()]) // public - .values_panic(["SCHEMA".into(), 1.into()]) // pg_catalog - .values_panic(["SCHEMA".into(), 1.into()]) // information_schema - .values_panic(["SCHEMA".into(), 1.into()]) // rw_catalog + .columns([Object::ObjType, Object::OwnerId, Object::DatabaseId]) + .values_panic(["DATABASE".into(), 1.into(), None::.into()]) + .values_panic(["SCHEMA".into(), 1.into(), 1.into()]) // public + .values_panic(["SCHEMA".into(), 1.into(), 1.into()]) // pg_catalog + .values_panic(["SCHEMA".into(), 1.into(), 1.into()]) // information_schema + .values_panic(["SCHEMA".into(), 1.into(), 1.into()]) // rw_catalog .to_owned(); // Since all tables are newly created, we assume that the initial object id of `dev` is 1 and the schemas' ids are 2, 3, 4, 5. @@ -836,11 +736,11 @@ impl MigrationTrait for Migration { .to_owned(); let insert_sys_schemas = Query::insert() .into_table(Schema::Table) - .columns([Schema::SchemaId, Schema::Name, Schema::DatabaseId]) - .values_panic([2.into(), "public".into(), 1.into()]) - .values_panic([3.into(), "pg_catalog".into(), 1.into()]) - .values_panic([4.into(), "information_schema".into(), 1.into()]) - .values_panic([5.into(), "rw_catalog".into(), 1.into()]) + .columns([Schema::SchemaId, Schema::Name]) + .values_panic([2.into(), "public".into()]) + .values_panic([3.into(), "pg_catalog".into()]) + .values_panic([4.into(), "information_schema".into()]) + .values_panic([5.into(), "rw_catalog".into()]) .to_owned(); manager.exec_stmt(insert_cluster_id).await?; @@ -962,7 +862,6 @@ enum Schema { Table, SchemaId, Name, - DatabaseId, } #[derive(DeriveIden)] @@ -1000,8 +899,6 @@ enum Table { Table, TableId, Name, - SchemaId, - DatabaseId, OptionalAssociatedSourceId, TableType, Columns, @@ -1028,8 +925,6 @@ enum Source { Table, SourceId, Name, - SchemaId, - DatabaseId, RowIdIndex, Columns, PkColumnIds, @@ -1046,8 +941,6 @@ enum Sink { Table, SinkId, Name, - SchemaId, - DatabaseId, Columns, PkColumnIds, DistributionKey, @@ -1065,8 +958,6 @@ enum Connection { Table, ConnectionId, Name, - SchemaId, - DatabaseId, Info, } @@ -1075,8 +966,6 @@ enum View { Table, ViewId, Name, - SchemaId, - DatabaseId, Properties, Sql, Columns, @@ -1087,8 +976,6 @@ enum Index { Table, IndexId, Name, - SchemaId, - DatabaseId, IndexTableId, PrimaryTableId, IndexItems, @@ -1100,8 +987,6 @@ enum Function { Table, FunctionId, Name, - SchemaId, - DatabaseId, ArgTypes, ReturnType, Language, @@ -1116,6 +1001,8 @@ enum Object { Oid, ObjType, OwnerId, + SchemaId, + DatabaseId, InitializedAt, CreatedAt, } diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs index 6370271551e88..d053a513a9671 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/src/model_v2/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; + use sea_orm::FromJsonQueryResult; use serde::{Deserialize, Serialize}; @@ -48,5 +50,34 @@ pub mod view; pub mod worker; pub mod worker_property; +pub type WorkerId = u32; +pub type TransactionId = u32; + +pub type ObjectId = u32; +pub type DatabaseId = ObjectId; +pub type SchemaId = ObjectId; +pub type TableId = ObjectId; +pub type SourceId = ObjectId; +pub type SinkId = ObjectId; +pub type IndexId = ObjectId; +pub type ViewId = ObjectId; +pub type FunctionId = ObjectId; +pub type ConnectionId = ObjectId; +pub type UserId = u32; + #[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] pub struct I32Array(pub Vec); + +#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] +pub struct DataType(pub risingwave_pb::data::DataType); + +#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] +pub struct DataTypeArray(pub Vec); + +#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Serialize, Deserialize, Default)] +pub struct FieldArray(pub Vec); + +impl Eq for FieldArray {} + +#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] +pub struct Property(pub HashMap); diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs index a2bc1164b631d..5048f93a483d9 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/src/model_v2/object.rs @@ -14,7 +14,9 @@ use sea_orm::entity::prelude::*; -#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +use crate::model_v2::{DatabaseId, ObjectId, SchemaId, UserId}; + +#[derive(Clone, Debug, PartialEq, Eq, Copy, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] pub enum ObjectType { #[sea_orm(string_value = "DATABASE")] @@ -37,13 +39,31 @@ pub enum ObjectType { Connection, } +impl ObjectType { + pub fn as_str(&self) -> &'static str { + match self { + ObjectType::Database => "database", + ObjectType::Schema => "schema", + ObjectType::Table => "table", + ObjectType::Source => "source", + ObjectType::Sink => "sink", + ObjectType::View => "view", + ObjectType::Index => "index", + ObjectType::Function => "function", + ObjectType::Connection => "connection", + } + } +} + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "object")] pub struct Model { #[sea_orm(primary_key)] - pub oid: i32, + pub oid: ObjectId, pub obj_type: ObjectType, - pub owner_id: i32, + pub owner_id: UserId, + pub schema_id: Option, + pub database_id: Option, pub initialized_at: DateTime, pub created_at: DateTime, } @@ -60,6 +80,22 @@ pub enum Relation { Function, #[sea_orm(has_many = "super::index::Entity")] Index, + #[sea_orm( + belongs_to = "Entity", + from = "Column::DatabaseId", + to = "Column::Oid", + on_update = "NoAction", + on_delete = "Cascade" + )] + SelfRef2, + #[sea_orm( + belongs_to = "Entity", + from = "Column::SchemaId", + to = "Column::Oid", + on_update = "NoAction", + on_delete = "Cascade" + )] + SelfRef1, #[sea_orm(has_many = "super::schema::Entity")] Schema, #[sea_orm(has_many = "super::sink::Entity")] @@ -73,7 +109,7 @@ pub enum Relation { from = "Column::OwnerId", to = "super::user::Column::UserId", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] User, #[sea_orm(has_many = "super::user_privilege::Entity")] diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/src/model_v2/object_dependency.rs index b1a0164ff0f4b..53800112a7370 100644 --- a/src/meta/src/model_v2/object_dependency.rs +++ b/src/meta/src/model_v2/object_dependency.rs @@ -14,13 +14,15 @@ use sea_orm::entity::prelude::*; +use crate::model_v2::{ObjectId, UserId}; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "object_dependency")] pub struct Model { #[sea_orm(primary_key)] pub id: i32, - pub oid: i32, - pub used_by: i32, + pub oid: ObjectId, + pub used_by: UserId, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/schema.rs b/src/meta/src/model_v2/schema.rs index 927989a1e6fb4..2c28665fd06f0 100644 --- a/src/meta/src/model_v2/schema.rs +++ b/src/meta/src/model_v2/schema.rs @@ -14,31 +14,18 @@ use sea_orm::entity::prelude::*; +use crate::model_v2::SchemaId; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "schema")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub schema_id: i32, + pub schema_id: SchemaId, pub name: String, - pub database_id: i32, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::connection::Entity")] - Connection, - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, - #[sea_orm(has_many = "super::function::Entity")] - Function, - #[sea_orm(has_many = "super::index::Entity")] - Index, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::SchemaId", @@ -47,38 +34,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm(has_many = "super::sink::Entity")] - Sink, - #[sea_orm(has_many = "super::source::Entity")] - Source, - #[sea_orm(has_many = "super::table::Entity")] - Table, - #[sea_orm(has_many = "super::view::Entity")] - View, -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Connection.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Function.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Index.def() - } } impl Related for Entity { @@ -87,28 +42,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Sink.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Source.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Table.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::View.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index 5501d05387edf..8c22a04a8fd01 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -14,7 +14,7 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::I32Array; +use crate::model_v2::{ConnectionId, I32Array, SinkId}; #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] @@ -31,10 +31,8 @@ pub enum SinkType { #[sea_orm(table_name = "sink")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub sink_id: i32, + pub sink_id: SinkId, pub name: String, - pub schema_id: i32, - pub database_id: i32, pub columns: Option, pub pk_column_ids: Option, pub distribution_key: Option, @@ -42,7 +40,7 @@ pub struct Model { pub sink_type: SinkType, pub properties: Option, pub definition: String, - pub connection_id: Option, + pub connection_id: Option, pub db_name: String, pub sink_from_name: String, } @@ -57,14 +55,6 @@ pub enum Relation { on_delete = "NoAction" )] Connection, - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::SinkId", @@ -73,14 +63,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, } impl Related for Entity { @@ -89,22 +71,10 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Object.def() } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs index f498bd643da8f..9bb6acc9382aa 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/src/model_v2/source.rs @@ -14,14 +14,14 @@ use sea_orm::entity::prelude::*; +use crate::model_v2::{ConnectionId, SourceId, TableId}; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "source")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub source_id: i32, + pub source_id: SourceId, pub name: String, - pub schema_id: i32, - pub database_id: i32, pub row_id_index: Option, pub columns: Option, pub pk_column_ids: Option, @@ -29,8 +29,8 @@ pub struct Model { pub definition: Option, pub source_info: Option, pub watermark_descs: Option, - pub optional_associated_table_id: Option, - pub connection_id: Option, + pub optional_associated_table_id: Option, + pub connection_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -43,14 +43,6 @@ pub enum Relation { on_delete = "NoAction" )] Connection, - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::SourceId", @@ -59,14 +51,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, #[sea_orm(has_many = "super::table::Entity")] Table, } @@ -77,24 +61,12 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Object.def() } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Table.def() diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs index 9b43272358d82..b2815eed7c8a0 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/src/model_v2/table.rs @@ -14,47 +14,50 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::I32Array; +use crate::model_v2::{I32Array, Property, SourceId, TableId}; + +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum TableType { + #[sea_orm(string_value = "TABLE")] + Table, + #[sea_orm(string_value = "MATERIALIZED_VIEW")] + MaterializedView, + #[sea_orm(string_value = "INDEX")] + Index, + #[sea_orm(string_value = "INTERNAL")] + Internal, +} #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "table")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub table_id: i32, + pub table_id: TableId, pub name: String, - pub schema_id: i32, - pub database_id: i32, - pub optional_associated_source_id: Option, - pub table_type: Option, - pub columns: Option, - pub pk: Option, - pub distribution_key: Option, - pub append_only: Option, - pub properties: Option, - pub fragment_id: Option, - pub vnode_col_index: Option, - pub value_indices: Option, - pub definition: Option, - pub handle_pk_conflict_behavior: Option, - pub read_prefix_len_hint: Option, - pub watermark_indices: Option, - pub dist_key_in_pk: Option, + pub optional_associated_source_id: Option, + pub table_type: TableType, + pub columns: Json, + pub pk: Json, + pub distribution_key: I32Array, + pub append_only: bool, + pub properties: Property, + pub fragment_id: i32, + pub vnode_col_index: I32Array, + pub value_indices: I32Array, + pub definition: String, + pub handle_pk_conflict_behavior: i32, + pub read_prefix_len_hint: i32, + pub watermark_indices: I32Array, + pub dist_key_in_pk: I32Array, pub dml_fragment_id: Option, pub cardinality: Option, - pub cleaned_by_watermark: Option, - pub version: Option, + pub cleaned_by_watermark: bool, + pub version: Json, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::fragment::Entity", from = "Column::DmlFragmentId", @@ -79,14 +82,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, #[sea_orm( belongs_to = "super::source::Entity", from = "Column::OptionalAssociatedSourceId", @@ -97,24 +92,12 @@ pub enum Relation { Source, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Object.def() } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Source.def() diff --git a/src/meta/src/model_v2/user.rs b/src/meta/src/model_v2/user.rs index b37316684470e..0e7ab4dd17876 100644 --- a/src/meta/src/model_v2/user.rs +++ b/src/meta/src/model_v2/user.rs @@ -14,11 +14,13 @@ use sea_orm::entity::prelude::*; +use crate::model_v2::UserId; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "user")] pub struct Model { #[sea_orm(primary_key)] - pub user_id: i32, + pub user_id: UserId, pub name: String, pub is_super: bool, pub can_create_db: bool, diff --git a/src/meta/src/model_v2/user_privilege.rs b/src/meta/src/model_v2/user_privilege.rs index 4079323904994..335f716cec1c8 100644 --- a/src/meta/src/model_v2/user_privilege.rs +++ b/src/meta/src/model_v2/user_privilege.rs @@ -14,14 +14,16 @@ use sea_orm::entity::prelude::*; +use crate::model_v2::{ObjectId, UserId}; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "user_privilege")] pub struct Model { #[sea_orm(primary_key)] pub id: i32, - pub user_id: i32, - pub oid: i32, - pub granted_by: i32, + pub user_id: UserId, + pub oid: ObjectId, + pub granted_by: UserId, pub actions: String, pub with_grant_option: bool, } diff --git a/src/meta/src/model_v2/view.rs b/src/meta/src/model_v2/view.rs index b0689e56afeef..5bad8593e0b72 100644 --- a/src/meta/src/model_v2/view.rs +++ b/src/meta/src/model_v2/view.rs @@ -12,31 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +use risingwave_pb::catalog::PbView; use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue; + +use crate::model_v2::{FieldArray, Property, ViewId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "view")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub view_id: i32, + pub view_id: ViewId, pub name: String, - pub schema_id: i32, - pub database_id: i32, - pub properties: Option, - pub sql: Option, - pub columns: Option, + pub properties: Property, + pub sql: String, + pub columns: FieldArray, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm( - belongs_to = "super::database::Entity", - from = "Column::DatabaseId", - to = "super::database::Column::DatabaseId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Database, #[sea_orm( belongs_to = "super::object::Entity", from = "Column::ViewId", @@ -45,20 +39,6 @@ pub enum Relation { on_delete = "Cascade" )] Object, - #[sea_orm( - belongs_to = "super::schema::Entity", - from = "Column::SchemaId", - to = "super::schema::Column::SchemaId", - on_update = "NoAction", - on_delete = "NoAction" - )] - Schema, -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } } impl Related for Entity { @@ -67,10 +47,16 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() +impl ActiveModelBehavior for ActiveModel {} + +impl From for ActiveModel { + fn from(view: PbView) -> Self { + Self { + view_id: ActiveValue::Set(view.id as _), + name: ActiveValue::Set(view.name), + properties: ActiveValue::Set(Property(view.properties)), + sql: ActiveValue::Set(view.sql), + columns: ActiveValue::Set(FieldArray(view.columns)), + } } } - -impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/worker.rs b/src/meta/src/model_v2/worker.rs index 6d786b0d5a860..08cdb2be34da1 100644 --- a/src/meta/src/model_v2/worker.rs +++ b/src/meta/src/model_v2/worker.rs @@ -13,6 +13,9 @@ // limitations under the License. use sea_orm::entity::prelude::*; + +use crate::model_v2::{TransactionId, WorkerId}; + #[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] pub enum WorkerType { @@ -41,12 +44,12 @@ pub enum WorkerStatus { #[sea_orm(table_name = "worker")] pub struct Model { #[sea_orm(primary_key)] - pub worker_id: i32, + pub worker_id: WorkerId, pub worker_type: WorkerType, pub host: String, pub port: i32, pub status: WorkerStatus, - pub transaction_id: Option, + pub transaction_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/worker_property.rs b/src/meta/src/model_v2/worker_property.rs index 862733d6f8544..8521cbed15ce2 100644 --- a/src/meta/src/model_v2/worker_property.rs +++ b/src/meta/src/model_v2/worker_property.rs @@ -14,13 +14,13 @@ use sea_orm::entity::prelude::*; -use crate::model_v2::I32Array; +use crate::model_v2::{I32Array, WorkerId}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "worker_property")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] - pub worker_id: i32, + pub worker_id: WorkerId, pub parallel_unit_ids: I32Array, pub is_streaming: bool, pub is_serving: bool, diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs index 722cce94b866a..2202e51ca6698 100644 --- a/src/meta/src/rpc/ddl_controller.rs +++ b/src/meta/src/rpc/ddl_controller.rs @@ -48,6 +48,7 @@ use crate::stream::{ }; use crate::{MetaError, MetaResult}; +#[derive(PartialEq)] pub enum DropMode { Restrict, Cascade, From bbd28521a296c007c1b761f8011ffc50c134cdcc Mon Sep 17 00:00:00 2001 From: xiangjinwu <17769960+xiangjinwu@users.noreply.github.com> Date: Fri, 20 Oct 2023 15:14:42 +0800 Subject: [PATCH 09/16] refactor(source): cleanup schema registry name strategy handling (#12968) --- src/connector/src/parser/avro/parser.rs | 28 ++++-- .../src/parser/debezium/avro_parser.rs | 23 ++--- src/connector/src/parser/protobuf/parser.rs | 18 ++-- .../src/parser/schema_registry/mod.rs | 89 +++++-------------- 4 files changed, 63 insertions(+), 95 deletions(-) diff --git a/src/connector/src/parser/avro/parser.rs b/src/connector/src/parser/avro/parser.rs index 0ce94cc1158eb..d583ef83a8227 100644 --- a/src/connector/src/parser/avro/parser.rs +++ b/src/connector/src/parser/avro/parser.rs @@ -126,22 +126,32 @@ impl AvroParserConfig { } else { None }; - let (subject_key, subject_value) = get_subject_by_strategy( + let subject_key = if enable_upsert { + Some(get_subject_by_strategy( + &avro_config.name_strategy, + avro_config.topic.as_str(), + avro_config.key_record_name.as_deref(), + true, + )?) + } else { + if let Some(name) = &avro_config.key_record_name { + return Err(RwError::from(ProtocolError(format!( + "key.message = {name} not used", + )))); + } + None + }; + let subject_value = get_subject_by_strategy( &avro_config.name_strategy, avro_config.topic.as_str(), - avro_config.key_record_name.as_deref(), avro_config.record_name.as_deref(), - enable_upsert, + false, )?; - tracing::debug!( - "infer key subject {}, value subject {}", - subject_key, - subject_value - ); + tracing::debug!("infer key subject {subject_key:?}, value subject {subject_value}"); Ok(Self { schema: resolver.get_by_subject_name(&subject_value).await?, - key_schema: if enable_upsert { + key_schema: if let Some(subject_key) = subject_key { Some(resolver.get_by_subject_name(&subject_key).await?) } else { None diff --git a/src/connector/src/parser/debezium/avro_parser.rs b/src/connector/src/parser/debezium/avro_parser.rs index a83b1443b5b1e..23161ba635b44 100644 --- a/src/connector/src/parser/debezium/avro_parser.rs +++ b/src/connector/src/parser/debezium/avro_parser.rs @@ -20,11 +20,14 @@ use apache_avro::{from_avro_datum, Schema}; use risingwave_common::error::ErrorCode::ProtocolError; use risingwave_common::error::{Result, RwError}; use risingwave_common::try_match_expand; +use risingwave_pb::catalog::PbSchemaRegistryNameStrategy; use risingwave_pb::plan_common::ColumnDesc; use crate::parser::avro::schema_resolver::ConfluentSchemaResolver; use crate::parser::avro::util::avro_schema_to_column_descs; -use crate::parser::schema_registry::{extract_schema_id, handle_sr_list, Client}; +use crate::parser::schema_registry::{ + extract_schema_id, get_subject_by_strategy, handle_sr_list, Client, +}; use crate::parser::unified::avro::{ avro_extract_field_schema, avro_schema_skip_union, AvroAccess, AvroParseOptions, }; @@ -108,18 +111,16 @@ impl DebeziumAvroParserConfig { let kafka_topic = &avro_config.topic; let url = handle_sr_list(schema_location)?; let client = Client::new(url, client_config)?; - let raw_schema = client - .get_schema_by_subject(format!("{}-key", &kafka_topic).as_str()) - .await?; - let key_schema = Schema::parse_str(&raw_schema.content) - .map_err(|e| RwError::from(ProtocolError(format!("Avro schema parse error {}", e))))?; - let resolver = ConfluentSchemaResolver::new(client); - let outer_schema = resolver - .get_by_subject_name(&format!("{}-value", kafka_topic)) - .await?; + + let name_strategy = &PbSchemaRegistryNameStrategy::TopicNameStrategyUnspecified; + let key_subject = get_subject_by_strategy(name_strategy, kafka_topic, None, true)?; + let val_subject = get_subject_by_strategy(name_strategy, kafka_topic, None, false)?; + let key_schema = resolver.get_by_subject_name(&key_subject).await?; + let outer_schema = resolver.get_by_subject_name(&val_subject).await?; + Ok(Self { - key_schema: Arc::new(key_schema), + key_schema, outer_schema, schema_resolver: Arc::new(resolver), }) diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs index 09763bcd6a49d..b07a7b214d2c9 100644 --- a/src/connector/src/parser/protobuf/parser.rs +++ b/src/connector/src/parser/protobuf/parser.rs @@ -83,19 +83,21 @@ impl ProtobufParserConfig { let message_name = &protobuf_config.message_name; let url = handle_sr_list(location.as_str())?; + if let Some(name) = protobuf_config.key_message_name { + // https://docs.confluent.io/platform/7.5/control-center/topics/schema.html#c3-schemas-best-practices-key-value-pairs + return Err(RwError::from(ProtocolError(format!( + "key.message = {name} not used. Protobuf key unsupported." + )))); + } let schema_bytes = if protobuf_config.use_schema_registry { - let (schema_key, schema_value) = get_subject_by_strategy( + let schema_value = get_subject_by_strategy( &protobuf_config.name_strategy, protobuf_config.topic.as_str(), - protobuf_config.key_message_name.as_deref(), Some(message_name.as_ref()), - protobuf_config.enable_upsert, + false, )?; - tracing::debug!( - "infer key subject {}, value subject {}", - schema_key, - schema_value, - ); + tracing::debug!("infer value subject {schema_value}"); + let client = Client::new(url, &protobuf_config.client_config)?; compile_file_descriptor_from_schema_registry(schema_value.as_str(), &client).await? } else { diff --git a/src/connector/src/parser/schema_registry/mod.rs b/src/connector/src/parser/schema_registry/mod.rs index 23a553b94dad7..85821f59b8115 100644 --- a/src/connector/src/parser/schema_registry/mod.rs +++ b/src/connector/src/parser/schema_registry/mod.rs @@ -32,77 +32,32 @@ pub fn name_strategy_from_str(value: &str) -> Option, record: Option<&str>, - require_key: bool, -) -> Result<(String, String), RwError> { - let build_error_lack_field = - |ns: &PbSchemaRegistryNameStrategy, expect: &[&str], got: &[Option<&str>]| -> RwError { - RwError::from(ProtocolError(format!( - "{:?} expect num-empty field {:?} but got {:?}", - ns.as_str_name(), - expect, - got - ))) - }; - let build_error_redundant_field = - |ns: &PbSchemaRegistryNameStrategy, expect: &[&str], got: &[Option<&str>]| -> RwError { - RwError::from(ProtocolError(format!( - "{:?} expect empty field {:?} but got {:?}", - ns.as_str_name(), - expect, - got - ))) - }; - match (name_strategy, require_key) { - (PbSchemaRegistryNameStrategy::TopicNameStrategyUnspecified, _) => { + is_key: bool, +) -> Result { + let build_error_lack_field = |ns: &PbSchemaRegistryNameStrategy, expect: &str| -> RwError { + RwError::from(ProtocolError(format!( + "{} expect num-empty field {}", + ns.as_str_name(), + expect, + ))) + }; + let record_option_name = if is_key { "key.message" } else { "message" }; + match name_strategy { + PbSchemaRegistryNameStrategy::TopicNameStrategyUnspecified => { // default behavior - Ok((format!("{}-key", topic), format!("{}-value", topic))) + let suffix = if is_key { "key" } else { "value" }; + Ok(format!("{topic}-{suffix}",)) } - (ns @ PbSchemaRegistryNameStrategy::RecordNameStrategy, true) => { - if let Some(record_name) = record && let Some(key_rec_name) = key_record_name { - Ok((key_rec_name.to_string(), record_name.to_string())) - } else { - Err(build_error_lack_field(ns, &["key.message","message"], &[key_record_name, record])) - } + ns @ PbSchemaRegistryNameStrategy::RecordNameStrategy => { + let record_name = + record.ok_or_else(|| build_error_lack_field(ns, record_option_name))?; + Ok(record_name.to_string()) } - (ns @ PbSchemaRegistryNameStrategy::RecordNameStrategy, false) => { - if key_record_name.is_some() { - return Err(build_error_redundant_field(ns, &["key.message"], &[key_record_name])); - } - if let Some(record_name) = record { - Ok(("".to_string(), record_name.to_string())) - } else { - Err(build_error_lack_field(ns, &["message"], &[record])) - } - } - (ns @ PbSchemaRegistryNameStrategy::TopicRecordNameStrategy, true) => { - if let Some(record_name) = record && let Some(key_rec_name) = key_record_name { - Ok(( - format!("{}-{}", topic, key_rec_name), - format!("{}-{}", topic, record_name), - )) - } else { - Err(build_error_lack_field( - ns, - &["topic", "key.message","message"], - &[Some(topic), key_record_name, record], - )) - } - } - (ns @ PbSchemaRegistryNameStrategy::TopicRecordNameStrategy, false) => { - if key_record_name.is_some() { - return Err(build_error_redundant_field(ns, &["key.message"], &[key_record_name])); - } - if let Some(record_name) = record { - Ok(("".to_string(), format!("{}-{}", topic, record_name))) - } else { - Err(build_error_lack_field( - ns, - &["topic","message"], - &[Some(topic), record], - )) - } + ns @ PbSchemaRegistryNameStrategy::TopicRecordNameStrategy => { + let record_name = + record.ok_or_else(|| build_error_lack_field(ns, record_option_name))?; + Ok(format!("{topic}-{record_name}")) } } } From 38a066c562cf6defb4d6d0bff0e3d488d9263132 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 20 Oct 2023 16:38:33 +0800 Subject: [PATCH 10/16] feat(memory): add jvm memory to compute node memory control (#12965) --- Cargo.lock | 1 + .../dashboards/risingwave-dev-dashboard.json | 2 +- .../dashboards/risingwave-user-dashboard.json | 2 +- grafana/risingwave-dev-dashboard.dashboard.py | 20 ++ grafana/risingwave-dev-dashboard.json | 2 +- grafana/risingwave-user-dashboard.json | 2 +- src/compute/Cargo.toml | 1 + .../src/memory_management/memory_manager.rs | 8 + src/compute/src/memory_management/mod.rs | 6 +- src/compute/src/memory_management/policy.rs | 24 ++- src/connector/src/sink/remote.rs | 6 +- .../src/source/cdc/enumerator/mod.rs | 2 +- src/connector/src/source/cdc/source/reader.rs | 6 +- src/jni_core/src/jvm_runtime.rs | 191 ++++++++++++------ .../src/executor/monitor/streaming_stats.rs | 18 ++ 15 files changed, 206 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68a19fffa8f47..47e8d5f09c30d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7220,6 +7220,7 @@ dependencies = [ "risingwave_common_service", "risingwave_connector", "risingwave_hummock_sdk", + "risingwave_jni_core", "risingwave_pb", "risingwave_rpc_client", "risingwave_source", diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json index dabc75f8a4d27..9b9bb59829d23 100644 --- a/docker/dashboards/risingwave-dev-dashboard.json +++ b/docker/dashboards/risingwave-dev-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, table_id) * on(actor_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":35,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":112},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":66,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":85,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the memory usage of mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size total - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the item counts in mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table counts total - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table count - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":97,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":118,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":156,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":174,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":194,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":201,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":204,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":213,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":221,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":240,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":253,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":271,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","query":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, table_id) * on(actor_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, table_id) * on(actor_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":35,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":112},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":66,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":85,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the memory usage of mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size total - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","query":"state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the item counts in mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table counts total - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table count - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","query":"state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":97,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":118,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":156,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","query":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":174,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":194,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":201,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":204,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":213,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":221,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":239,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":242,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":255,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":273,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} diff --git a/docker/dashboards/risingwave-user-dashboard.json b/docker/dashboards/risingwave-user-dashboard.json index 41445ec1e30fb..63e3ed095ac69 100644 --- a/docker/dashboards/risingwave-user-dashboard.json +++ b/docker/dashboards/risingwave-user-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0) + ((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_safe_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":42},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":50},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":51},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":53},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":54},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":55},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_name}}","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id {{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0) + ((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_safe_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0) + ((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_safe_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":42},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":50},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":51},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":53},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":54},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":55},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py index 183ae7bf0be6f..e22837410866b 100644 --- a/grafana/risingwave-dev-dashboard.dashboard.py +++ b/grafana/risingwave-dev-dashboard.dashboard.py @@ -3220,6 +3220,26 @@ def section_memory_manager(outer_panels): ), ], ), + panels.timeseries_memory( + "The allocated memory of jvm", + "", + [ + panels.target( + f"{metric('jvm_allocated_bytes')}", + "", + ), + ], + ), + panels.timeseries_memory( + "The active memory of jvm", + "", + [ + panels.target( + f"{metric('jvm_active_bytes')}", + "", + ), + ], + ), panels.timeseries_ms( "LRU manager diff between current watermark and evicted watermark time (ms) for actors", "", diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json index dabc75f8a4d27..9b9bb59829d23 100644 --- a/grafana/risingwave-dev-dashboard.json +++ b/grafana/risingwave-dev-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, table_id) * on(actor_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":35,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":112},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":66,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":85,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the memory usage of mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size total - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the item counts in mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table counts total - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table count - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":97,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":118,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":156,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":174,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":194,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":201,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":204,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":213,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":221,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":240,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":253,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":271,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","query":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, table_id) * on(actor_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, table_id) * on(actor_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":35,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"distinct agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top n appendonly total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup executor total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":112},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":66,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":85,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the memory usage of mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size total - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table size - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","query":"state_store_mem_table_memory_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the item counts in mem_table.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table counts total - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem_table count - table id {{table_id}} instance id {{instance_id}} {{job}} @ {{instance}}","metric":"","query":"state_store_mem_table_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":97,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":118,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":156,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","query":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":174,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":194,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":201,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":204,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":213,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":221,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":239,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":242,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":255,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":273,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} diff --git a/grafana/risingwave-user-dashboard.json b/grafana/risingwave-user-dashboard.json index 41445ec1e30fb..63e3ed095ac69 100644 --- a/grafana/risingwave-user-dashboard.json +++ b/grafana/risingwave-user-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0) + ((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_safe_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":42},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":50},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":51},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":53},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":54},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":55},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_name}}","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id {{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0) + ((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_safe_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0) + ((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_safe_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":42},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":50},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":51},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":53},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":54},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":55},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index 93e4e643dcf04..624e901c6b7ae 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -33,6 +33,7 @@ risingwave_common_heap_profiling = { workspace = true } risingwave_common_service = { workspace = true } risingwave_connector = { workspace = true } risingwave_hummock_sdk = { workspace = true } +risingwave_jni_core = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_source = { workspace = true } diff --git a/src/compute/src/memory_management/memory_manager.rs b/src/compute/src/memory_management/memory_manager.rs index 3e8e0fb6c0a47..721b39046572a 100644 --- a/src/compute/src/memory_management/memory_manager.rs +++ b/src/compute/src/memory_management/memory_manager.rs @@ -79,6 +79,8 @@ impl GlobalMemoryManager { let mut memory_control_stats = MemoryControlStats { jemalloc_allocated_bytes: 0, jemalloc_active_bytes: 0, + jvm_allocated_bytes: 0, + jvm_active_bytes: 0, lru_watermark_step: 0, lru_watermark_time_ms: Epoch::physical_now(), lru_physical_now_ms: Epoch::physical_now(), @@ -122,6 +124,12 @@ impl GlobalMemoryManager { self.metrics .jemalloc_active_bytes .set(memory_control_stats.jemalloc_active_bytes as i64); + self.metrics + .jvm_allocated_bytes + .set(memory_control_stats.jvm_allocated_bytes as i64); + self.metrics + .jvm_active_bytes + .set(memory_control_stats.jvm_active_bytes as i64); } } } diff --git a/src/compute/src/memory_management/mod.rs b/src/compute/src/memory_management/mod.rs index 0323cb2ddc093..b6811935a1e9d 100644 --- a/src/compute/src/memory_management/mod.rs +++ b/src/compute/src/memory_management/mod.rs @@ -49,6 +49,8 @@ pub const STORAGE_DEFAULT_HIGH_PRIORITY_BLOCK_CACHE_RATIO: usize = 70; pub struct MemoryControlStats { pub jemalloc_allocated_bytes: usize, pub jemalloc_active_bytes: usize, + pub jvm_allocated_bytes: usize, + pub jvm_active_bytes: usize, pub lru_watermark_step: u64, pub lru_watermark_time_ms: u64, pub lru_physical_now_ms: u64, @@ -68,9 +70,9 @@ pub trait MemoryControl: Send + Sync + std::fmt::Debug { } pub fn build_memory_control_policy(total_memory_bytes: usize) -> MemoryControlRef { - use self::policy::JemallocMemoryControl; + use self::policy::JemallocAndJvmMemoryControl; - Box::new(JemallocMemoryControl::new(total_memory_bytes)) + Box::new(JemallocAndJvmMemoryControl::new(total_memory_bytes)) } /// `DummyPolicy` is used for operarting systems other than Linux. It does nothing as memory control diff --git a/src/compute/src/memory_management/policy.rs b/src/compute/src/memory_management/policy.rs index 71e85c6149258..1eb5ff9832600 100644 --- a/src/compute/src/memory_management/policy.rs +++ b/src/compute/src/memory_management/policy.rs @@ -17,15 +17,16 @@ use std::sync::Arc; use risingwave_batch::task::BatchManager; use risingwave_common::util::epoch::Epoch; +use risingwave_jni_core::jvm_runtime::load_jvm_memory_stats; use risingwave_stream::task::LocalStreamManager; use tikv_jemalloc_ctl::{epoch as jemalloc_epoch, stats as jemalloc_stats}; use super::{MemoryControl, MemoryControlStats}; -/// `JemallocMemoryControl` is a memory control policy that uses jemalloc statistics to control. It -/// assumes that most memory is used by streaming engine and does memory control over LRU watermark -/// based on jemalloc statistics. -pub struct JemallocMemoryControl { +/// `JemallocAndJvmMemoryControl` is a memory control policy that uses jemalloc statistics and +/// jvm memory statistics and to control. It assumes that most memory is used by streaming engine +/// and does memory control over LRU watermark based on jemalloc statistics and jvm memory statistics. +pub struct JemallocAndJvmMemoryControl { threshold_stable: usize, threshold_graceful: usize, threshold_aggressive: usize, @@ -35,7 +36,7 @@ pub struct JemallocMemoryControl { jemalloc_active_mib: jemalloc_stats::active_mib, } -impl JemallocMemoryControl { +impl JemallocAndJvmMemoryControl { const THRESHOLD_AGGRESSIVE: f64 = 0.9; const THRESHOLD_GRACEFUL: f64 = 0.8; const THRESHOLD_STABLE: f64 = 0.7; @@ -81,7 +82,7 @@ impl JemallocMemoryControl { } } -impl std::fmt::Debug for JemallocMemoryControl { +impl std::fmt::Debug for JemallocAndJvmMemoryControl { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("JemallocMemoryControl") .field("threshold_stable", &self.threshold_stable) @@ -91,7 +92,7 @@ impl std::fmt::Debug for JemallocMemoryControl { } } -impl MemoryControl for JemallocMemoryControl { +impl MemoryControl for JemallocAndJvmMemoryControl { fn apply( &self, interval_ms: u32, @@ -105,13 +106,15 @@ impl MemoryControl for JemallocMemoryControl { prev_memory_stats.jemalloc_active_bytes, ); + let (jvm_allocated_bytes, jvm_active_bytes) = load_jvm_memory_stats(); + // Streaming memory control // // We calculate the watermark of the LRU cache, which provides hints for streaming executors // on cache eviction. Here we do the calculation based on jemalloc statistics. let (lru_watermark_step, lru_watermark_time_ms, lru_physical_now) = calculate_lru_watermark( - jemalloc_allocated_bytes, + jemalloc_allocated_bytes + jvm_allocated_bytes, self.threshold_stable, self.threshold_graceful, self.threshold_aggressive, @@ -124,6 +127,8 @@ impl MemoryControl for JemallocMemoryControl { MemoryControlStats { jemalloc_allocated_bytes, jemalloc_active_bytes, + jvm_allocated_bytes, + jvm_active_bytes, lru_watermark_step, lru_watermark_time_ms, lru_physical_now_ms: lru_physical_now, @@ -141,7 +146,8 @@ fn calculate_lru_watermark( ) -> (u64, u64, u64) { let mut watermark_time_ms = prev_memory_stats.lru_watermark_time_ms; let last_step = prev_memory_stats.lru_watermark_step; - let last_used_memory_bytes = prev_memory_stats.jemalloc_allocated_bytes; + let last_used_memory_bytes = + prev_memory_stats.jemalloc_allocated_bytes + prev_memory_stats.jvm_allocated_bytes; // The watermark calculation works in the following way: // diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs index b694696842746..ad182e734a33a 100644 --- a/src/connector/src/sink/remote.rs +++ b/src/connector/src/sink/remote.rs @@ -148,7 +148,7 @@ impl Sink for RemoteSink { }).try_collect()?; let mut env = JVM - .as_ref() + .get_or_init() .map_err(|err| SinkError::Internal(err.into()))? .attach_current_thread() .map_err(|err| SinkError::Internal(err.into()))?; @@ -380,7 +380,7 @@ impl RemoteSinkWriterInner { }; std::thread::spawn(move || { - let mut env = JVM.as_ref().unwrap().attach_current_thread().unwrap(); + let mut env = JVM.get_or_init().unwrap().attach_current_thread().unwrap(); let result = env.call_static_method( "com/risingwave/connector/JniSinkWriterHandler", @@ -625,7 +625,7 @@ impl RemoteCoordinator { }; std::thread::spawn(move || { - let mut env = JVM.as_ref().unwrap().attach_current_thread().unwrap(); + let mut env = JVM.get_or_init().unwrap().attach_current_thread().unwrap(); let result = env.call_static_method( "com/risingwave/connector/JniSinkCoordinatorHandler", diff --git a/src/connector/src/source/cdc/enumerator/mod.rs b/src/connector/src/source/cdc/enumerator/mod.rs index 0b98005b270f4..e88440bc876e1 100644 --- a/src/connector/src/source/cdc/enumerator/mod.rs +++ b/src/connector/src/source/cdc/enumerator/mod.rs @@ -69,7 +69,7 @@ where SourceType::from(T::source_type()) ); - let mut env = JVM.as_ref()?.attach_current_thread()?; + let mut env = JVM.get_or_init()?.attach_current_thread()?; let validate_source_request = ValidateSourceRequest { source_id: context.info.source_id as u64, diff --git a/src/connector/src/source/cdc/source/reader.rs b/src/connector/src/source/cdc/source/reader.rs index 0c2fb2b3b2684..7410834ce1daa 100644 --- a/src/connector/src/source/cdc/source/reader.rs +++ b/src/connector/src/source/cdc/source/reader.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::str::FromStr; -use std::sync::LazyLock; use anyhow::{anyhow, Result}; use async_trait::async_trait; @@ -121,7 +120,8 @@ impl CommonSplitReader for CdcSplitReader { let (tx, mut rx) = mpsc::channel(DEFAULT_CHANNEL_SIZE); - LazyLock::force(&JVM).as_ref()?; + // Force init, because we don't want to see initialization failure in the following thread. + JVM.get_or_init()?; let get_event_stream_request = GetEventStreamRequest { source_id: self.source_id, @@ -135,7 +135,7 @@ impl CommonSplitReader for CdcSplitReader { let source_type = get_event_stream_request.source_type.to_string(); std::thread::spawn(move || { - let mut env = JVM.as_ref().unwrap().attach_current_thread().unwrap(); + let mut env = JVM.get_or_init().unwrap().attach_current_thread().unwrap(); let get_event_stream_request_bytes = env .byte_array_from_slice(&Message::encode_to_vec(&get_event_stream_request)) diff --git a/src/jni_core/src/jvm_runtime.rs b/src/jni_core/src/jvm_runtime.rs index 5d93a122ab4a1..e60c17c6ec930 100644 --- a/src/jni_core/src/jvm_runtime.rs +++ b/src/jni_core/src/jvm_runtime.rs @@ -16,86 +16,110 @@ use core::option::Option::Some; use std::ffi::c_void; use std::fs; use std::path::Path; -use std::sync::LazyLock; +use std::sync::OnceLock; +use jni::objects::JValueOwned; use jni::strings::JNIString; use jni::{InitArgsBuilder, JNIVersion, JavaVM, NativeMethod}; use risingwave_common::error::{ErrorCode, RwError}; use risingwave_common::util::resource_util::memory::system_memory_available_bytes; -pub static JVM: LazyLock> = LazyLock::new(|| { - let libs_path = if let Ok(libs_path) = std::env::var("CONNECTOR_LIBS_PATH") { - libs_path - } else { - return Err(ErrorCode::InternalError( - "environment variable CONNECTOR_LIBS_PATH is not specified".to_string(), - ) - .into()); - }; +/// Use 10% of compute total memory by default. Compute node uses 0.7 * system memory by default. +const DEFAULT_MEMORY_PROPORTION: f64 = 0.07; - let dir = Path::new(&libs_path); +pub static JVM: JavaVmWrapper = JavaVmWrapper::new(); - if !dir.is_dir() { - return Err(ErrorCode::InternalError(format!( - "CONNECTOR_LIBS_PATH \"{}\" is not a directory", - libs_path - )) - .into()); +pub struct JavaVmWrapper(OnceLock>); + +impl JavaVmWrapper { + const fn new() -> Self { + Self(OnceLock::new()) } - let mut class_vec = vec![]; + pub fn get(&self) -> Option<&Result> { + self.0.get() + } - if let Ok(entries) = fs::read_dir(dir) { - for entry in entries.flatten() { - let entry_path = entry.path(); - if entry_path.file_name().is_some() { - let path = std::fs::canonicalize(entry_path)?; - class_vec.push(path.to_str().unwrap().to_string()); - } - } - } else { - return Err(ErrorCode::InternalError(format!( - "failed to read CONNECTOR_LIBS_PATH \"{}\"", - libs_path - )) - .into()); + pub fn get_or_init(&self) -> Result<&JavaVM, &RwError> { + self.0.get_or_init(Self::inner_new).as_ref() } - let jvm_heap_size = if let Ok(heap_size) = std::env::var("JVM_HEAP_SIZE") { - heap_size - } else { - // Use 10% of total memory by default - // TODO: should use compute-node's total_memory_bytes - format!("{}", system_memory_available_bytes() / 10) - }; - - // Build the VM properties - let args_builder = InitArgsBuilder::new() - // Pass the JNI API version (default is 8) - .version(JNIVersion::V8) - .option("-ea") - .option("-Dis_embedded_connector=true") - .option(format!("-Djava.class.path={}", class_vec.join(":"))) - .option(format!("-Xmx{}", jvm_heap_size)); - - tracing::info!("JVM args: {:?}", args_builder); - let jvm_args = args_builder.build().unwrap(); - - // Create a new VM - let jvm = match JavaVM::new(jvm_args) { - Err(err) => { - tracing::error!("fail to new JVM {:?}", err); - return Err(ErrorCode::InternalError("fail to new JVM".to_string()).into()); + fn inner_new() -> Result { + let libs_path = if let Ok(libs_path) = std::env::var("CONNECTOR_LIBS_PATH") { + libs_path + } else { + return Err(ErrorCode::InternalError( + "environment variable CONNECTOR_LIBS_PATH is not specified".to_string(), + ) + .into()); + }; + + let dir = Path::new(&libs_path); + + if !dir.is_dir() { + return Err(ErrorCode::InternalError(format!( + "CONNECTOR_LIBS_PATH \"{}\" is not a directory", + libs_path + )) + .into()); + } + + let mut class_vec = vec![]; + + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries.flatten() { + let entry_path = entry.path(); + if entry_path.file_name().is_some() { + let path = std::fs::canonicalize(entry_path)?; + class_vec.push(path.to_str().unwrap().to_string()); + } + } + } else { + return Err(ErrorCode::InternalError(format!( + "failed to read CONNECTOR_LIBS_PATH \"{}\"", + libs_path + )) + .into()); } - Ok(jvm) => jvm, - }; - tracing::info!("initialize JVM successfully"); + let jvm_heap_size = if let Ok(heap_size) = std::env::var("JVM_HEAP_SIZE") { + heap_size + } else { + format!( + "{}", + (system_memory_available_bytes() as f64 * DEFAULT_MEMORY_PROPORTION) as usize + ) + }; + + // Build the VM properties + let args_builder = InitArgsBuilder::new() + // Pass the JNI API version (default is 8) + .version(JNIVersion::V8) + .option("-ea") + .option("-Dis_embedded_connector=true") + .option(format!("-Djava.class.path={}", class_vec.join(":"))) + .option("-Xms16m") + .option(format!("-Xmx{}", jvm_heap_size)); + + tracing::info!("JVM args: {:?}", args_builder); + let jvm_args = args_builder.build().unwrap(); + + // Create a new VM + let jvm = match JavaVM::new(jvm_args) { + Err(err) => { + tracing::error!("fail to new JVM {:?}", err); + return Err(ErrorCode::InternalError("fail to new JVM".to_string()).into()); + } + Ok(jvm) => jvm, + }; + + tracing::info!("initialize JVM successfully"); - register_native_method_for_jvm(&jvm).unwrap(); + register_native_method_for_jvm(&jvm).unwrap(); - Ok(jvm) -}); + Ok(jvm) + } +} pub fn register_native_method_for_jvm(jvm: &JavaVM) -> Result<(), jni::errors::Error> { let mut env = jvm @@ -134,3 +158,44 @@ pub fn register_native_method_for_jvm(jvm: &JavaVM) -> Result<(), jni::errors::E tracing::info!("register native methods for jvm successfully"); Ok(()) } + +/// Load JVM memory statistics from the runtime. If JVM is not initialized or fail to initialize, return zero. +pub fn load_jvm_memory_stats() -> (usize, usize) { + if let Some(jvm) = JVM.get() { + match jvm { + Ok(jvm) => { + let mut env = jvm.attach_current_thread().unwrap(); + let runtime_instance = env + .call_static_method( + "java/lang/Runtime", + "getRuntime", + "()Ljava/lang/Runtime;", + &[], + ) + .unwrap(); + + let runtime_instance = match runtime_instance { + JValueOwned::Object(o) => o, + _ => unreachable!(), + }; + + let total_memory = env + .call_method(runtime_instance.as_ref(), "totalMemory", "()J", &[]) + .unwrap() + .j() + .unwrap(); + + let free_memory = env + .call_method(runtime_instance, "freeMemory", "()J", &[]) + .unwrap() + .j() + .unwrap(); + + (total_memory as usize, (total_memory - free_memory) as usize) + } + Err(_) => (0, 0), + } + } else { + (0, 0) + } +} diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs index 459e9cbaa439c..a31727e76639f 100644 --- a/src/stream/src/executor/monitor/streaming_stats.rs +++ b/src/stream/src/executor/monitor/streaming_stats.rs @@ -156,6 +156,8 @@ pub struct StreamingMetrics { pub lru_evicted_watermark_time_ms: GenericGaugeVec, pub jemalloc_allocated_bytes: IntGauge, pub jemalloc_active_bytes: IntGauge, + pub jvm_allocated_bytes: IntGauge, + pub jvm_active_bytes: IntGauge, /// User compute error reporting pub user_compute_error_count: GenericCounterVec, @@ -865,6 +867,20 @@ impl StreamingMetrics { ) .unwrap(); + let jvm_allocated_bytes = register_int_gauge_with_registry!( + "jvm_allocated_bytes", + "The allocated jvm memory", + registry + ) + .unwrap(); + + let jvm_active_bytes = register_int_gauge_with_registry!( + "jvm_active_bytes", + "The active jvm memory", + registry + ) + .unwrap(); + let user_compute_error_count = register_int_counter_vec_with_registry!( "user_compute_error_count", "Compute errors in the system, queryable by tags", @@ -994,6 +1010,8 @@ impl StreamingMetrics { lru_evicted_watermark_time_ms, jemalloc_allocated_bytes, jemalloc_active_bytes, + jvm_allocated_bytes, + jvm_active_bytes, user_compute_error_count, user_source_reader_error_count, materialize_cache_hit_count, From 852b4275f0d6ca7e34bc75c0a5244206a3569a2c Mon Sep 17 00:00:00 2001 From: August Date: Fri, 20 Oct 2023 19:03:14 +0800 Subject: [PATCH 11/16] fix: fix table object foreign key in init migration (#12978) --- src/meta/src/model_v2/migration/src/m20230908_072257_init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index 35d1ab29020e6..a50e3bcba6e69 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -472,7 +472,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Table::Version).json().not_null()) .foreign_key( &mut ForeignKey::create() - .name("FK_view_object_id") + .name("FK_table_object_id") .from(Table::Table, Table::TableId) .to(Object::Table, Object::Oid) .on_delete(ForeignKeyAction::Cascade) From 37d0a79d046e48d7175153d800e53ae465462402 Mon Sep 17 00:00:00 2001 From: stonepage <40830455+st1page@users.noreply.github.com> Date: Fri, 20 Oct 2023 19:15:54 +0800 Subject: [PATCH 12/16] feat: concurrent batch scan (#12825) Co-authored-by: lmatz --- e2e_test/streaming/watermark.slt | 12 +++---- .../src/table/batch_table/storage_table.rs | 31 +++++-------------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/e2e_test/streaming/watermark.slt b/e2e_test/streaming/watermark.slt index 4bbb5e1d83885..d1a29b88188cf 100644 --- a/e2e_test/streaming/watermark.slt +++ b/e2e_test/streaming/watermark.slt @@ -25,7 +25,7 @@ sleep 10s skipif in-memory query TI -select * from mv1; +select * from mv1 order by 2; ---- 2023-05-06 16:51:00 1 2023-05-06 16:51:00 2 @@ -33,16 +33,16 @@ select * from mv1; skipif in-memory query TI -select * from mv2; +select * from mv2 order by 2; ---- 2023-05-06 16:51:00 1 -2023-05-06 16:51:00 2 -2023-05-06 16:51:00 3 2023-05-06 16:51:00 1 -2023-05-06 16:51:00 2 -2023-05-06 16:51:00 3 2023-05-06 16:51:00 1 2023-05-06 16:51:00 2 +2023-05-06 16:51:00 2 +2023-05-06 16:51:00 2 +2023-05-06 16:51:00 3 +2023-05-06 16:51:00 3 2023-05-06 16:51:00 3 statement ok diff --git a/src/storage/src/table/batch_table/storage_table.rs b/src/storage/src/table/batch_table/storage_table.rs index e4eb65b8b9f42..f784c4a5e7ca2 100644 --- a/src/storage/src/table/batch_table/storage_table.rs +++ b/src/storage/src/table/batch_table/storage_table.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::assert_matches::assert_matches; use std::ops::Bound::{self, Excluded, Included, Unbounded}; use std::ops::{Index, RangeBounds}; use std::sync::Arc; @@ -46,7 +45,7 @@ use crate::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew}; use crate::row_serde::{find_columns_by_ids, ColumnMapping}; use crate::store::{PrefetchOptions, ReadOptions}; use crate::table::merge_sort::merge_sort; -use crate::table::{compute_vnode, Distribution, KeyedRow, TableIter, DEFAULT_VNODE}; +use crate::table::{compute_vnode, Distribution, KeyedRow, TableIter}; use crate::StateStore; /// [`StorageTableInner`] is the interface accessing relational data in KV(`StateStore`) with @@ -418,24 +417,7 @@ impl StorageTableInner { _ => CachePolicy::Fill(CachePriority::High), }; - let raw_key_ranges = if !ordered - && matches!(encoded_key_range.start_bound(), Unbounded) - && matches!(encoded_key_range.end_bound(), Unbounded) - { - // If the range is unbounded and order is not required, we can create a single iterator - // for each continuous vnode range. - - // In this case, the `vnode_hint` must be default for singletons and `None` for - // distributed tables. - assert_eq!(vnode_hint.unwrap_or(DEFAULT_VNODE), DEFAULT_VNODE); - - Either::Left(self.vnodes.vnode_ranges().map(|r| { - let start = Included(Bytes::copy_from_slice(&r.start().to_be_bytes()[..])); - let end = end_bound_of_prefix(&r.end().to_be_bytes()); - assert_matches!(end, Excluded(_) | Unbounded); - (start, end) - })) - } else { + let raw_key_ranges = { // Vnodes that are set and should be accessed. let vnodes = match vnode_hint { // If `vnode_hint` is set, we can only access this single vnode. @@ -443,9 +425,7 @@ impl StorageTableInner { // Otherwise, we need to access all vnodes of this table. None => Either::Right(self.vnodes.iter_vnodes()), }; - Either::Right( - vnodes.map(|vnode| prefixed_range(encoded_key_range.clone(), &vnode.to_be_bytes())), - ) + vnodes.map(|vnode| prefixed_range(encoded_key_range.clone(), &vnode.to_be_bytes())) }; // For each key range, construct an iterator. @@ -493,7 +473,10 @@ impl StorageTableInner { 0 => unreachable!(), 1 => iterators.into_iter().next().unwrap(), // Concat all iterators if not to preserve order. - _ if !ordered => futures::stream::iter(iterators).flatten(), + _ if !ordered => { + futures::stream::iter(iterators.into_iter().map(Box::pin).collect_vec()) + .flatten_unordered(1024) + } // Merge all iterators if to preserve order. _ => merge_sort(iterators.into_iter().map(Box::pin).collect()), }; From fa66cbda67c9996821bb4f1d7d03026fada0a712 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 20 Oct 2023 06:48:14 -0500 Subject: [PATCH 13/16] refactor(optimizer): clean up dead code for optimizer v2 (#12973) Signed-off-by: Bugen Zhao --- src/frontend/src/optimizer/mod.rs | 1 - src/frontend/src/optimizer/plan_node/batch.rs | 5 + .../src/optimizer/plan_node/generic/join.rs | 78 +- .../src/optimizer/plan_node/logical_agg.rs | 1 - .../src/optimizer/plan_node/logical_join.rs | 1 - src/frontend/src/optimizer/plan_node/mod.rs | 3 - .../src/optimizer/plan_node/plan_base.rs | 2 + .../src/optimizer/plan_node/plan_tree_node.rs | 41 - .../optimizer/plan_node/plan_tree_node_v2.rs | 124 --- .../src/optimizer/plan_node/stream.rs | 772 +----------------- .../src/optimizer/plan_node/stream_dedup.rs | 2 +- .../optimizer/plan_node/stream_delta_join.rs | 1 - .../src/optimizer/plan_node/stream_derive.rs | 630 -------------- .../plan_node/stream_eowc_over_window.rs | 1 - .../optimizer/plan_node/stream_exchange.rs | 1 - .../src/optimizer/plan_node/stream_expand.rs | 1 - .../src/optimizer/plan_node/stream_filter.rs | 1 - .../optimizer/plan_node/stream_hash_join.rs | 6 +- .../optimizer/plan_node/stream_hop_window.rs | 1 - .../src/optimizer/plan_node/stream_project.rs | 1 - .../optimizer/plan_node/stream_project_set.rs | 1 - .../optimizer/plan_node/stream_row_id_gen.rs | 1 - .../plan_node/stream_stateless_simple_agg.rs | 1 - .../plan_node/stream_temporal_join.rs | 1 - .../src/optimizer/plan_node/stream_union.rs | 1 - .../src/optimizer/property/distribution.rs | 18 - 26 files changed, 94 insertions(+), 1602 deletions(-) delete mode 100644 src/frontend/src/optimizer/plan_node/plan_tree_node_v2.rs delete mode 100644 src/frontend/src/optimizer/plan_node/stream_derive.rs diff --git a/src/frontend/src/optimizer/mod.rs b/src/frontend/src/optimizer/mod.rs index aaab8ebde3dcd..b4238f57b1f52 100644 --- a/src/frontend/src/optimizer/mod.rs +++ b/src/frontend/src/optimizer/mod.rs @@ -57,7 +57,6 @@ use self::plan_visitor::{has_batch_exchange, CardinalityVisitor}; use self::property::{Cardinality, RequiredDist}; use self::rule::*; use crate::catalog::table_catalog::{TableType, TableVersion}; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::{ BatchExchange, PlanNodeType, PlanTreeNode, RewriteExprsRecursive, }; diff --git a/src/frontend/src/optimizer/plan_node/batch.rs b/src/frontend/src/optimizer/plan_node/batch.rs index 2ac1e278f7d8b..d62a85095d21c 100644 --- a/src/frontend/src/optimizer/plan_node/batch.rs +++ b/src/frontend/src/optimizer/plan_node/batch.rs @@ -15,6 +15,11 @@ use super::generic::GenericPlanRef; use crate::optimizer::property::Order; +/// A subtrait of [`GenericPlanRef`] for batch plans. +/// +/// Due to the lack of refactoring, all plan nodes currently implement this trait +/// through [`super::PlanBase`]. One may still use this trait as a bound for +/// expecting a batch plan, in contrast to [`GenericPlanRef`]. pub trait BatchPlanRef: GenericPlanRef { fn order(&self) -> &Order; } diff --git a/src/frontend/src/optimizer/plan_node/generic/join.rs b/src/frontend/src/optimizer/plan_node/generic/join.rs index 95bee8413f56f..87c03cc14c8c9 100644 --- a/src/frontend/src/optimizer/plan_node/generic/join.rs +++ b/src/frontend/src/optimizer/plan_node/generic/join.rs @@ -12,15 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use itertools::EitherOrBoth; -use risingwave_common::catalog::Schema; +use itertools::{EitherOrBoth, Itertools}; +use risingwave_common::catalog::{Field, Schema}; +use risingwave_common::types::DataType; +use risingwave_common::util::sort_util::OrderType; use risingwave_pb::plan_common::JoinType; use super::{EqJoinPredicate, GenericPlanNode, GenericPlanRef}; use crate::expr::ExprRewriter; use crate::optimizer::optimizer_context::OptimizerContextRef; +use crate::optimizer::plan_node::stream; +use crate::optimizer::plan_node::utils::TableCatalogBuilder; use crate::optimizer::property::FunctionalDependencySet; use crate::utils::{ColIndexMapping, ColIndexMappingRewriteExt, Condition}; +use crate::TableCatalog; /// [`Join`] combines two relations according to some condition. /// @@ -65,6 +70,75 @@ impl Join { } } +impl Join { + /// Return stream hash join internal table catalog and degree table catalog. + pub fn infer_internal_and_degree_table_catalog( + input: &PlanRef, + join_key_indices: Vec, + dk_indices_in_jk: Vec, + ) -> (TableCatalog, TableCatalog, Vec) { + let schema = input.schema(); + + let internal_table_dist_keys = dk_indices_in_jk + .iter() + .map(|idx| join_key_indices[*idx]) + .collect_vec(); + + let degree_table_dist_keys = dk_indices_in_jk.clone(); + + // The pk of hash join internal and degree table should be join_key + input_pk. + let join_key_len = join_key_indices.len(); + let mut pk_indices = join_key_indices; + + // dedup the pk in dist key.. + let mut deduped_input_pk_indices = vec![]; + for input_pk_idx in input.stream_key().unwrap() { + if !pk_indices.contains(input_pk_idx) + && !deduped_input_pk_indices.contains(input_pk_idx) + { + deduped_input_pk_indices.push(*input_pk_idx); + } + } + + pk_indices.extend(deduped_input_pk_indices.clone()); + + // Build internal table + let mut internal_table_catalog_builder = + TableCatalogBuilder::new(input.ctx().with_options().internal_table_subset()); + let internal_columns_fields = schema.fields().to_vec(); + + internal_columns_fields.iter().for_each(|field| { + internal_table_catalog_builder.add_column(field); + }); + pk_indices.iter().for_each(|idx| { + internal_table_catalog_builder.add_order_column(*idx, OrderType::ascending()) + }); + + // Build degree table. + let mut degree_table_catalog_builder = + TableCatalogBuilder::new(input.ctx().with_options().internal_table_subset()); + + let degree_column_field = Field::with_name(DataType::Int64, "_degree"); + + pk_indices.iter().enumerate().for_each(|(order_idx, idx)| { + degree_table_catalog_builder.add_column(&internal_columns_fields[*idx]); + degree_table_catalog_builder.add_order_column(order_idx, OrderType::ascending()); + }); + degree_table_catalog_builder.add_column(°ree_column_field); + degree_table_catalog_builder + .set_value_indices(vec![degree_table_catalog_builder.columns().len() - 1]); + + internal_table_catalog_builder.set_dist_key_in_pk(dk_indices_in_jk.clone()); + degree_table_catalog_builder.set_dist_key_in_pk(dk_indices_in_jk); + + ( + internal_table_catalog_builder.build(internal_table_dist_keys, join_key_len), + degree_table_catalog_builder.build(degree_table_dist_keys, join_key_len), + deduped_input_pk_indices, + ) + } +} + impl GenericPlanNode for Join { fn schema(&self) -> Schema { let left_schema = self.left.schema(); diff --git a/src/frontend/src/optimizer/plan_node/logical_agg.rs b/src/frontend/src/optimizer/plan_node/logical_agg.rs index 0ab848eb29f64..cffe453879bde 100644 --- a/src/frontend/src/optimizer/plan_node/logical_agg.rs +++ b/src/frontend/src/optimizer/plan_node/logical_agg.rs @@ -31,7 +31,6 @@ use crate::expr::{ WindowFunction, }; use crate::optimizer::plan_node::generic::GenericPlanNode; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::{ gen_filter_and_pushdown, BatchSortAgg, ColumnPruningContext, LogicalDedup, LogicalProject, PredicatePushdownContext, RewriteStreamContext, ToStreamContext, diff --git a/src/frontend/src/optimizer/plan_node/logical_join.rs b/src/frontend/src/optimizer/plan_node/logical_join.rs index e76a99fd15e21..cfc49a1da3353 100644 --- a/src/frontend/src/optimizer/plan_node/logical_join.rs +++ b/src/frontend/src/optimizer/plan_node/logical_join.rs @@ -31,7 +31,6 @@ use super::{ }; use crate::expr::{CollectInputRef, Expr, ExprImpl, ExprRewriter, ExprType, InputRef}; use crate::optimizer::plan_node::generic::DynamicFilter; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::utils::IndicesDisplay; use crate::optimizer::plan_node::{ BatchHashJoin, BatchLookupJoin, BatchNestedLoopJoin, ColumnPruningContext, EqJoinPredicate, diff --git a/src/frontend/src/optimizer/plan_node/mod.rs b/src/frontend/src/optimizer/plan_node/mod.rs index 0a9e2a5dee517..188787c93b8c0 100644 --- a/src/frontend/src/optimizer/plan_node/mod.rs +++ b/src/frontend/src/optimizer/plan_node/mod.rs @@ -617,8 +617,6 @@ impl dyn PlanNode { } mod plan_base; -#[macro_use] -mod plan_tree_node_v2; pub use plan_base::*; #[macro_use] mod plan_tree_node; @@ -641,7 +639,6 @@ pub use merge_eq_nodes::*; pub mod batch; pub mod generic; pub mod stream; -pub mod stream_derive; pub use generic::{PlanAggCall, PlanAggCallDisplay}; diff --git a/src/frontend/src/optimizer/plan_node/plan_base.rs b/src/frontend/src/optimizer/plan_node/plan_base.rs index 22e239a7369d5..e9a5bf26885bf 100644 --- a/src/frontend/src/optimizer/plan_node/plan_base.rs +++ b/src/frontend/src/optimizer/plan_node/plan_base.rs @@ -85,11 +85,13 @@ impl stream::StreamPlanRef for PlanBase { self.emit_on_window_close } } + impl batch::BatchPlanRef for PlanBase { fn order(&self) -> &Order { &self.order } } + impl PlanBase { pub fn new_logical( ctx: OptimizerContextRef, diff --git a/src/frontend/src/optimizer/plan_node/plan_tree_node.rs b/src/frontend/src/optimizer/plan_node/plan_tree_node.rs index e1435a6b7b20d..0c46d91f7a566 100644 --- a/src/frontend/src/optimizer/plan_node/plan_tree_node.rs +++ b/src/frontend/src/optimizer/plan_node/plan_tree_node.rs @@ -109,19 +109,6 @@ macro_rules! impl_plan_tree_node_for_leaf { self.clone().into() } } - - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $leaf_node_type { - type PlanRef = crate::optimizer::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[crate::optimizer::PlanRef; 2]> { - smallvec::smallvec![] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - assert!(inputs.next().is_none(), "expect exactly no input"); - self.clone() - } - } }; } @@ -141,20 +128,6 @@ macro_rules! impl_plan_tree_node_for_unary { self.clone_with_input(inputs[0].clone()).into() } } - - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $unary_node_type { - type PlanRef = crate::optimizer::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[crate::optimizer::PlanRef; 2]> { - smallvec::smallvec![self.input()] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - let input = inputs.next().expect("expect exactly 1 input"); - assert!(inputs.next().is_none(), "expect exactly 1 input"); - self.clone_with_input(input).into() - } - } }; } @@ -174,19 +147,5 @@ macro_rules! impl_plan_tree_node_for_binary { .into() } } - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $binary_node_type { - type PlanRef = crate::optimizer::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[crate::optimizer::PlanRef; 2]> { - smallvec::smallvec![self.left(), self.right()] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - let left = inputs.next().expect("expect exactly 2 input"); - let right = inputs.next().expect("expect exactly 2 input"); - assert!(inputs.next().is_none(), "expect exactly 2 input"); - self.clone_with_left_right(left, right).into() - } - } }; } diff --git a/src/frontend/src/optimizer/plan_node/plan_tree_node_v2.rs b/src/frontend/src/optimizer/plan_node/plan_tree_node_v2.rs deleted file mode 100644 index e598c7dd61caa..0000000000000 --- a/src/frontend/src/optimizer/plan_node/plan_tree_node_v2.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use smallvec::SmallVec; - -pub trait PlanTreeNodeV2 { - type PlanRef; - - fn inputs(&self) -> SmallVec<[Self::PlanRef; 2]>; - fn clone_with_inputs(&self, inputs: impl Iterator) -> Self; -} - -macro_rules! impl_plan_tree_node_v2_for_stream_leaf_node { - ($node_type:ident) => { - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $node_type { - type PlanRef = crate::optimizer::plan_node::stream::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[Self::PlanRef; 2]> { - smallvec::smallvec![] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - assert!(inputs.next().is_none(), "expect exactly no input"); - self.clone() - } - } - }; -} - -macro_rules! impl_plan_tree_node_v2_for_stream_unary_node { - ($node_type:ident, $input_field:ident) => { - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $node_type { - type PlanRef = crate::optimizer::plan_node::stream::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[Self::PlanRef; 2]> { - smallvec::smallvec![self.$input_field.clone()] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - let mut new = self.clone(); - new.$input_field = inputs.next().expect("expect exactly 1 input"); - assert!(inputs.next().is_none(), "expect exactly 1 input"); - new.clone() - } - } - }; -} - -// macro_rules! impl_plan_tree_node_v2_for_stream_binary_node { -// ($node_type:ident, $first_input_field:ident, $second_input_field:ident) => { -// impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $node_type { -// type PlanRef = crate::optimizer::plan_node::stream::PlanRef; - -// fn inputs(&self) -> smallvec::SmallVec<[Self::PlanRef; 2]> { -// smallvec::smallvec![ -// self.$first_input_field.clone(), -// self.$second_input_field.clone() -// ] -// } - -// fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self -// { let mut new = self.clone(); -// new.$first_input_field = inputs.next().expect("expect exactly 2 input"); -// new.$second_input_field = inputs.next().expect("expect exactly 2 input"); -// assert!(inputs.next().is_none(), "expect exactly 2 input"); -// new.clone() -// } -// } -// }; -// } - -macro_rules! impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating { - ($node_type:ident, $core_field:ident, $input_field:ident) => { - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $node_type { - type PlanRef = crate::optimizer::plan_node::stream::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[Self::PlanRef; 2]> { - smallvec::smallvec![self.$core_field.$input_field.clone()] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - let mut new = self.clone(); - new.$core_field.$input_field = inputs.next().expect("expect exactly 1 input"); - assert!(inputs.next().is_none(), "expect exactly 1 input"); - new.clone() - } - } - }; -} - -macro_rules! impl_plan_tree_node_v2_for_stream_binary_node_with_core_delegating { - ($node_type:ident, $core_field:ident, $first_input_field:ident, $second_input_field:ident) => { - impl crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2 for $node_type { - type PlanRef = crate::optimizer::plan_node::stream::PlanRef; - - fn inputs(&self) -> smallvec::SmallVec<[Self::PlanRef; 2]> { - smallvec::smallvec![ - self.$core_field.$first_input_field.clone(), - self.$core_field.$second_input_field.clone() - ] - } - - fn clone_with_inputs(&self, mut inputs: impl Iterator) -> Self { - let mut new = self.clone(); - new.$core_field.$first_input_field = inputs.next().expect("expect exactly 2 input"); - new.$core_field.$second_input_field = - inputs.next().expect("expect exactly 2 input"); - assert!(inputs.next().is_none(), "expect exactly 2 input"); - new.clone() - } - } - }; -} diff --git a/src/frontend/src/optimizer/plan_node/stream.rs b/src/frontend/src/optimizer/plan_node/stream.rs index 36b2c77d22aed..2edf997bf91fd 100644 --- a/src/frontend/src/optimizer/plan_node/stream.rs +++ b/src/frontend/src/optimizer/plan_node/stream.rs @@ -12,774 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use educe::Educe; -use generic::PlanAggCall; -use itertools::Itertools; -use pb::stream_node as pb_node; -use risingwave_common::catalog::{ColumnDesc, Field, Schema}; -use risingwave_common::types::DataType; -use risingwave_common::util::sort_util::{ColumnOrder, OrderType}; -use risingwave_connector::sink::catalog::desc::SinkDesc; -use risingwave_pb::stream_plan as pb; -use smallvec::SmallVec; - -use super::generic::{GenericPlanNode, GenericPlanRef}; -use super::utils::TableCatalogBuilder; -use super::{generic, EqJoinPredicate, PlanNodeId}; -use crate::expr::{Expr, ExprImpl}; -use crate::optimizer::optimizer_context::OptimizerContextRef; -use crate::optimizer::plan_node::plan_tree_node_v2::PlanTreeNodeV2; -use crate::optimizer::property::{Distribution, FunctionalDependencySet}; -use crate::stream_fragmenter::BuildFragmentGraphState; -use crate::TableCatalog; - -macro_rules! impl_node { -($base:ident, $($t:ident),*) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub enum Node { - $($t(Box<$t>),)* - } - pub type PlanOwned = ($base, Node); - pub type PlanRef = std::rc::Rc; - $( - impl From<$t> for PlanRef { - fn from(o: $t) -> PlanRef { - std::rc::Rc::new((o.to_stream_base(), Node::$t(Box::new(o)))) - } - } - )* - impl PlanTreeNodeV2 for PlanRef { - type PlanRef = PlanRef; - - fn inputs(&self) -> SmallVec<[Self::PlanRef; 2]> { - match &self.1 { - $(Node::$t(inner) => inner.inputs(),)* - } - } - fn clone_with_inputs(&self, inputs: impl Iterator) -> Self { - match &self.1 { - $(Node::$t(inner) => inner.clone_with_inputs(inputs).into(),)* - } - } - - } -}; -} - -pub trait StreamPlanNode: GenericPlanNode { - fn distribution(&self) -> Distribution; - fn append_only(&self) -> bool; - fn emit_on_window_close(&self) -> bool; - fn to_stream_base(&self) -> PlanBase { - let ctx = self.ctx(); - PlanBase { - id: ctx.next_plan_node_id(), - ctx, - schema: self.schema(), - stream_key: self.stream_key(), - dist: self.distribution(), - append_only: self.append_only(), - emit_on_window_close: self.emit_on_window_close(), - } - } -} +use super::generic::GenericPlanRef; +use crate::optimizer::property::Distribution; +/// A subtrait of [`GenericPlanRef`] for stream plans. +/// +/// Due to the lack of refactoring, all plan nodes currently implement this trait +/// through [`super::PlanBase`]. One may still use this trait as a bound for +/// expecting a stream plan, in contrast to [`GenericPlanRef`]. pub trait StreamPlanRef: GenericPlanRef { fn distribution(&self) -> &Distribution; fn append_only(&self) -> bool; fn emit_on_window_close(&self) -> bool; } - -impl generic::GenericPlanRef for PlanRef { - fn schema(&self) -> &Schema { - &self.0.schema - } - - fn stream_key(&self) -> Option<&[usize]> { - self.0.stream_key.as_deref() - } - - fn ctx(&self) -> OptimizerContextRef { - self.0.ctx.clone() - } - - fn functional_dependency(&self) -> &FunctionalDependencySet { - self.0.functional_dependency() - } -} - -impl generic::GenericPlanRef for PlanBase { - fn schema(&self) -> &Schema { - &self.schema - } - - fn stream_key(&self) -> Option<&[usize]> { - self.stream_key.as_deref() - } - - fn ctx(&self) -> OptimizerContextRef { - self.ctx.clone() - } - - fn functional_dependency(&self) -> &FunctionalDependencySet { - todo!() - } -} - -impl StreamPlanRef for PlanBase { - fn distribution(&self) -> &Distribution { - &self.dist - } - - fn append_only(&self) -> bool { - self.append_only - } - - fn emit_on_window_close(&self) -> bool { - self.emit_on_window_close - } -} - -impl StreamPlanRef for PlanRef { - fn distribution(&self) -> &Distribution { - &self.0.dist - } - - fn append_only(&self) -> bool { - self.0.append_only - } - - fn emit_on_window_close(&self) -> bool { - self.0.emit_on_window_close - } -} - -/// Implements [`generic::Join`] with delta join. It requires its two -/// inputs to be indexes. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct DeltaJoin { - pub core: generic::Join, - - /// The join condition must be equivalent to `logical.on`, but separated into equal and - /// non-equal parts to facilitate execution later - pub eq_join_predicate: EqJoinPredicate, -} -impl_plan_tree_node_v2_for_stream_binary_node_with_core_delegating!(DeltaJoin, core, left, right); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct DynamicFilter { - pub core: generic::DynamicFilter, -} -impl_plan_tree_node_v2_for_stream_binary_node_with_core_delegating!( - DynamicFilter, - core, - left, - right -); -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Exchange { - pub dist: Distribution, - pub input: PlanRef, -} -impl_plan_tree_node_v2_for_stream_unary_node!(Exchange, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Expand { - pub core: generic::Expand, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(Expand, core, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Filter { - pub core: generic::Filter, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(Filter, core, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct SimpleAgg { - pub core: generic::Agg, - /// The index of `count(*)` in `agg_calls`. - row_count_idx: usize, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(SimpleAgg, core, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct GroupTopN { - pub core: generic::TopN, - /// an optional column index which is the vnode of each row computed by the input's consistent - /// hash distribution - pub vnode_col_idx: Option, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(GroupTopN, core, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HashAgg { - pub core: generic::Agg, - /// An optional column index which is the vnode of each row computed by the input's consistent - /// hash distribution. - vnode_col_idx: Option, - /// The index of `count(*)` in `agg_calls`. - row_count_idx: usize, - /// Whether to emit output only when the window is closed by watermark. - emit_on_window_close: bool, - /// The watermark column that Emit-On-Window-Close behavior is based on. - window_col_idx: Option, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(HashAgg, core, input); - -/// Implements [`generic::Join`] with hash table. It builds a hash table -/// from inner (right-side) relation and probes with data from outer (left-side) relation to -/// get output rows. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HashJoin { - pub core: generic::Join, - - /// The join condition must be equivalent to `logical.on`, but separated into equal and - /// non-equal parts to facilitate execution later - pub eq_join_predicate: EqJoinPredicate, - - /// Whether can optimize for append-only stream. - /// It is true if input of both side is append-only - pub is_append_only: bool, -} -impl_plan_tree_node_v2_for_stream_binary_node_with_core_delegating!(HashJoin, core, left, right); - -impl HashJoin { - /// Return hash join internal table catalog and degree table catalog. - pub fn infer_internal_and_degree_table_catalog( - input: &impl StreamPlanRef, - join_key_indices: Vec, - dk_indices_in_jk: Vec, - ) -> (TableCatalog, TableCatalog, Vec) { - let schema = input.schema(); - - let internal_table_dist_keys = dk_indices_in_jk - .iter() - .map(|idx| join_key_indices[*idx]) - .collect_vec(); - - let degree_table_dist_keys = dk_indices_in_jk.clone(); - - // The pk of hash join internal and degree table should be join_key + input_pk. - let join_key_len = join_key_indices.len(); - let mut pk_indices = join_key_indices; - - // dedup the pk in dist key.. - let mut deduped_input_pk_indices = vec![]; - for input_pk_idx in input.stream_key().unwrap() { - if !pk_indices.contains(input_pk_idx) - && !deduped_input_pk_indices.contains(input_pk_idx) - { - deduped_input_pk_indices.push(*input_pk_idx); - } - } - - pk_indices.extend(deduped_input_pk_indices.clone()); - - // Build internal table - let mut internal_table_catalog_builder = - TableCatalogBuilder::new(input.ctx().with_options().internal_table_subset()); - let internal_columns_fields = schema.fields().to_vec(); - - internal_columns_fields.iter().for_each(|field| { - internal_table_catalog_builder.add_column(field); - }); - pk_indices.iter().for_each(|idx| { - internal_table_catalog_builder.add_order_column(*idx, OrderType::ascending()) - }); - - // Build degree table. - let mut degree_table_catalog_builder = - TableCatalogBuilder::new(input.ctx().with_options().internal_table_subset()); - - let degree_column_field = Field::with_name(DataType::Int64, "_degree"); - - pk_indices.iter().enumerate().for_each(|(order_idx, idx)| { - degree_table_catalog_builder.add_column(&internal_columns_fields[*idx]); - degree_table_catalog_builder.add_order_column(order_idx, OrderType::ascending()); - }); - degree_table_catalog_builder.add_column(°ree_column_field); - degree_table_catalog_builder - .set_value_indices(vec![degree_table_catalog_builder.columns().len() - 1]); - - internal_table_catalog_builder.set_dist_key_in_pk(dk_indices_in_jk.clone()); - degree_table_catalog_builder.set_dist_key_in_pk(dk_indices_in_jk); - - ( - internal_table_catalog_builder.build(internal_table_dist_keys, join_key_len), - degree_table_catalog_builder.build(degree_table_dist_keys, join_key_len), - deduped_input_pk_indices, - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HopWindow { - pub core: generic::HopWindow, - window_start_exprs: Vec, - window_end_exprs: Vec, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(HopWindow, core, input); - -/// [`IndexScan`] is a virtual plan node to represent a stream table scan. It will be converted -/// to chain + merge node (for upstream materialize) + batch table scan when converting to `MView` -/// creation request. Compared with [`TableScan`], it will reorder columns, and the chain node -/// doesn't allow rearrange. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct IndexScan { - pub core: generic::Scan, - pub batch_plan_id: PlanNodeId, -} -impl_plan_tree_node_v2_for_stream_leaf_node!(IndexScan); - -/// Stateless simple agg. -/// -/// Should only be used for stateless agg, including `sum`, `count` and *append-only* `min`/`max`. -/// -/// The output of `StatelessSimpleAgg` doesn't have pk columns, so the result can only be used by -/// `SimpleAgg` with `ManagedValueState`s. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct StatelessSimpleAgg { - pub core: generic::Agg, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(StatelessSimpleAgg, core, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Materialize { - /// Child of Materialize plan - pub input: PlanRef, - pub table: TableCatalog, -} -impl_plan_tree_node_v2_for_stream_unary_node!(Materialize, input); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ProjectSet { - pub core: generic::ProjectSet, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(ProjectSet, core, input); - -/// `Project` implements [`super::LogicalProject`] to evaluate specified expressions on input -/// rows. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Project { - pub core: generic::Project, - watermark_derivations: Vec<(usize, usize)>, - merge_chunk: bool, - nondecreasing_exprs: Vec, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(Project, core, input); - -/// [`Sink`] represents a table/connector sink at the very end of the graph. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Sink { - pub input: PlanRef, - pub sink_desc: SinkDesc, -} -impl_plan_tree_node_v2_for_stream_unary_node!(Sink, input); -/// [`Source`] represents a table/connector source at the very beginning of the graph. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Source { - pub core: generic::Source, -} -impl_plan_tree_node_v2_for_stream_leaf_node!(Source); - -/// `TableScan` is a virtual plan node to represent a stream table scan. It will be converted -/// to chain + merge node (for upstream materialize) + batch table scan when converting to `MView` -/// creation request. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TableScan { - pub core: generic::Scan, - pub batch_plan_id: PlanNodeId, -} -impl_plan_tree_node_v2_for_stream_leaf_node!(TableScan); - -/// `TopN` implements [`super::LogicalTopN`] to find the top N elements with a heap -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TopN { - pub core: generic::TopN, -} -impl_plan_tree_node_v2_for_stream_unary_node_with_core_delegating!(TopN, core, input); - -#[derive(Clone, Debug, Educe)] -#[educe(PartialEq, Eq, Hash)] -pub struct PlanBase { - #[educe(PartialEq(ignore))] - #[educe(Hash(ignore))] - pub id: PlanNodeId, - #[educe(PartialEq(ignore))] - #[educe(Hash(ignore))] - pub ctx: OptimizerContextRef, - pub schema: Schema, - pub stream_key: Option>, - #[educe(PartialEq(ignore))] - #[educe(Hash(ignore))] - pub dist: Distribution, - pub append_only: bool, - pub emit_on_window_close: bool, -} - -impl_node!( - PlanBase, - Exchange, - DynamicFilter, - DeltaJoin, - Expand, - Filter, - SimpleAgg, - GroupTopN, - HashAgg, - HashJoin, - HopWindow, - IndexScan, - StatelessSimpleAgg, - Materialize, - ProjectSet, - Project, - Sink, - Source, - TableScan, - TopN -); - -use pb_node::PbNodeBody; -#[allow(dead_code)] -pub fn to_stream_prost_body( - (base, core): &PlanOwned, - state: &mut BuildFragmentGraphState, -) -> PbNodeBody { - use pb::*; - match core { - Node::TableScan(_) => todo!(), - Node::IndexScan(_) => todo!(), - // ^ need standalone implementations - Node::Exchange(_) => PbNodeBody::Exchange(ExchangeNode { - strategy: Some(DispatchStrategy { - r#type: match &base.dist { - Distribution::HashShard(_) => DispatcherType::Hash, - Distribution::Single => DispatcherType::Simple, - Distribution::Broadcast => DispatcherType::Broadcast, - _ => panic!("Do not allow Any or AnyShard in serialization process"), - } as i32, - dist_key_indices: match &base.dist { - Distribution::HashShard(keys) => keys.iter().map(|&num| num as u32).collect(), - _ => vec![], - }, - output_indices: (0..base.schema().len() as u32).collect(), - }), - }), - Node::DynamicFilter(me) => { - use generic::dynamic_filter::*; - let me = &me.core; - let condition = me - .predicate() - .as_expr_unless_true() - .map(|x| x.to_expr_proto()); - let left_table = infer_left_internal_table_catalog(base, me.left_index()) - .with_id(state.gen_table_id_wrapped()); - let right_table = infer_right_internal_table_catalog(&me.right().0) - .with_id(state.gen_table_id_wrapped()); - PbNodeBody::DynamicFilter(DynamicFilterNode { - left_key: me.left_index() as u32, - condition, - left_table: Some(left_table.to_internal_table_prost()), - right_table: Some(right_table.to_internal_table_prost()), - }) - } - Node::DeltaJoin(me) => { - let (_, left_node) = &*me.core.left; - let (_, right_node) = &*me.core.right; - fn cast(node: &Node) -> &IndexScan { - match node { - Node::IndexScan(scan) => scan, - _ => unreachable!(), - } - } - let left_table = cast(left_node); - let right_table = cast(right_node); - let left_table_desc = &*left_table.core.table_desc; - let right_table_desc = &*right_table.core.table_desc; - - // TODO: add a separate delta join node in proto, or move fragmenter to frontend so that - // we don't need an intermediate representation. - PbNodeBody::DeltaIndexJoin(DeltaIndexJoinNode { - join_type: me.core.join_type as i32, - left_key: me - .eq_join_predicate - .left_eq_indexes() - .iter() - .map(|v| *v as i32) - .collect(), - right_key: me - .eq_join_predicate - .right_eq_indexes() - .iter() - .map(|v| *v as i32) - .collect(), - condition: me - .eq_join_predicate - .other_cond() - .as_expr_unless_true() - .map(|x| x.to_expr_proto()), - left_table_id: left_table_desc.table_id.table_id(), - right_table_id: right_table_desc.table_id.table_id(), - left_info: Some(ArrangementInfo { - arrange_key_orders: left_table_desc.arrange_key_orders_protobuf(), - column_descs: left_table - .core - .column_descs() - .iter() - .map(ColumnDesc::to_protobuf) - .collect(), - table_desc: Some(left_table_desc.to_protobuf()), - }), - right_info: Some(ArrangementInfo { - arrange_key_orders: right_table_desc.arrange_key_orders_protobuf(), - column_descs: right_table - .core - .column_descs() - .iter() - .map(ColumnDesc::to_protobuf) - .collect(), - table_desc: Some(right_table_desc.to_protobuf()), - }), - output_indices: me.core.output_indices.iter().map(|&x| x as u32).collect(), - }) - } - Node::Expand(me) => { - use pb::expand_node::Subset; - - let me = &me.core; - PbNodeBody::Expand(ExpandNode { - column_subsets: me - .column_subsets - .iter() - .map(|subset| { - let column_indices = subset.iter().map(|&key| key as u32).collect(); - Subset { column_indices } - }) - .collect(), - }) - } - Node::Filter(me) => { - let me = &me.core; - PbNodeBody::Filter(FilterNode { - search_condition: Some(ExprImpl::from(me.predicate.clone()).to_expr_proto()), - }) - } - Node::SimpleAgg(me) => { - let intermediate_state_table = me.core.infer_intermediate_state_table(base, None, None); - let agg_states = me.core.infer_stream_agg_state(base, None, None); - let distinct_dedup_tables = me.core.infer_distinct_dedup_tables(base, None, None); - - PbNodeBody::SimpleAgg(SimpleAggNode { - agg_calls: me - .core - .agg_calls - .iter() - .map(PlanAggCall::to_protobuf) - .collect(), - row_count_index: me.row_count_idx as u32, - distribution_key: base - .dist - .dist_column_indices() - .iter() - .map(|&idx| idx as u32) - .collect(), - is_append_only: me.core.input.0.append_only, - agg_call_states: agg_states - .into_iter() - .map(|s| s.into_prost(state)) - .collect(), - intermediate_state_table: Some( - intermediate_state_table - .with_id(state.gen_table_id_wrapped()) - .to_internal_table_prost(), - ), - distinct_dedup_tables: distinct_dedup_tables - .into_iter() - .map(|(key_idx, table)| (key_idx as u32, table.to_internal_table_prost())) - .collect(), - }) - } - Node::GroupTopN(me) => { - let input = &me.core.input.0; - let table = me - .core - .infer_internal_table_catalog( - input.schema(), - input.ctx(), - input.stream_key().unwrap(), - me.vnode_col_idx, - ) - .with_id(state.gen_table_id_wrapped()); - let group_topn_node = GroupTopNNode { - limit: me.core.limit_attr.limit(), - offset: me.core.offset, - with_ties: me.core.limit_attr.with_ties(), - group_key: me.core.group_key.iter().map(|idx| *idx as u32).collect(), - table: Some(table.to_internal_table_prost()), - order_by: me.core.order.to_protobuf(), - }; - - PbNodeBody::GroupTopN(group_topn_node) - } - Node::HashAgg(me) => { - let intermediate_state_table = - me.core - .infer_intermediate_state_table(base, me.vnode_col_idx, me.window_col_idx); - let agg_states = - me.core - .infer_stream_agg_state(base, me.vnode_col_idx, me.window_col_idx); - let distinct_dedup_tables = - me.core - .infer_distinct_dedup_tables(base, me.vnode_col_idx, me.window_col_idx); - - PbNodeBody::HashAgg(HashAggNode { - group_key: me.core.group_key.indices().map(|idx| idx as u32).collect(), - agg_calls: me - .core - .agg_calls - .iter() - .map(PlanAggCall::to_protobuf) - .collect(), - row_count_index: me.row_count_idx as u32, - is_append_only: me.core.input.0.append_only, - agg_call_states: agg_states - .into_iter() - .map(|s| s.into_prost(state)) - .collect(), - intermediate_state_table: Some( - intermediate_state_table - .with_id(state.gen_table_id_wrapped()) - .to_internal_table_prost(), - ), - distinct_dedup_tables: distinct_dedup_tables - .into_iter() - .map(|(key_idx, table)| (key_idx as u32, table.to_internal_table_prost())) - .collect(), - emit_on_window_close: me.emit_on_window_close(), - }) - } - Node::HashJoin(_) => { - unreachable!(); - } - Node::HopWindow(me) => { - let window_start_exprs = me - .window_start_exprs - .clone() - .iter() - .map(|x| x.to_expr_proto()) - .collect(); - let window_end_exprs = me - .window_end_exprs - .clone() - .iter() - .map(|x| x.to_expr_proto()) - .collect(); - let me = &me.core; - PbNodeBody::HopWindow(HopWindowNode { - time_col: me.time_col.index() as _, - window_slide: Some(me.window_slide.into()), - window_size: Some(me.window_size.into()), - output_indices: me.output_indices.iter().map(|&x| x as u32).collect(), - window_start_exprs, - window_end_exprs, - }) - } - Node::StatelessSimpleAgg(me) => { - let me = &me.core; - PbNodeBody::StatelessSimpleAgg(SimpleAggNode { - agg_calls: me.agg_calls.iter().map(PlanAggCall::to_protobuf).collect(), - row_count_index: u32::MAX, // this is not used - distribution_key: base - .dist - .dist_column_indices() - .iter() - .map(|&idx| idx as u32) - .collect(), - agg_call_states: vec![], - intermediate_state_table: None, - is_append_only: me.input.0.append_only, - distinct_dedup_tables: Default::default(), - }) - } - Node::Materialize(me) => { - PbNodeBody::Materialize(MaterializeNode { - // We don't need table id for materialize node in frontend. The id will be generated - // on meta catalog service. - table_id: 0, - column_orders: me.table.pk().iter().map(ColumnOrder::to_protobuf).collect(), - table: Some(me.table.to_internal_table_prost()), - }) - } - Node::ProjectSet(_) => { - unreachable!() - } - Node::Project(me) => PbNodeBody::Project(ProjectNode { - select_list: me.core.exprs.iter().map(|x| x.to_expr_proto()).collect(), - watermark_input_cols: me - .watermark_derivations - .iter() - .map(|(x, _)| *x as u32) - .collect(), - watermark_output_cols: me - .watermark_derivations - .iter() - .map(|(_, y)| *y as u32) - .collect(), - nondecreasing_exprs: me.nondecreasing_exprs.iter().map(|i| *i as u32).collect(), - }), - Node::Sink(me) => PbNodeBody::Sink(SinkNode { - sink_desc: Some(me.sink_desc.to_proto()), - table: None, // TODO: Refactor sink to have a generic core. - log_store_type: SinkLogStoreType::InMemoryLogStore as i32, - }), - Node::Source(me) => { - // TODO(kwannoel): Is branch used, seems to be a duplicate of stream_source? - let rate_limit = me.ctx().session_ctx().config().get_streaming_rate_limit(); - let me = &me.core.catalog; - let source_inner = me.as_ref().map(|me| StreamSource { - source_id: me.id, - source_name: me.name.clone(), - state_table: Some( - generic::Source::infer_internal_table_catalog() - .with_id(state.gen_table_id_wrapped()) - .to_internal_table_prost(), - ), - info: Some(me.info.clone()), - row_id_index: me.row_id_index.map(|index| index as _), - columns: me.columns.iter().map(|c| c.to_protobuf()).collect(), - properties: me.properties.clone().into_iter().collect(), - rate_limit, - }); - PbNodeBody::Source(SourceNode { source_inner }) - } - Node::TopN(me) => { - let input = &me.core.input.0; - let me = &me.core; - let topn_node = TopNNode { - limit: me.limit_attr.limit(), - offset: me.offset, - with_ties: me.limit_attr.with_ties(), - table: Some( - me.infer_internal_table_catalog( - input.schema(), - input.ctx(), - input - .stream_key() - .expect("should always have a stream key in the stream plan but not"), - None, - ) - .with_id(state.gen_table_id_wrapped()) - .to_internal_table_prost(), - ), - order_by: me.order.to_protobuf(), - }; - if me.input.0.append_only { - PbNodeBody::AppendOnlyTopN(topn_node) - } else { - PbNodeBody::TopN(topn_node) - } - } - } -} diff --git a/src/frontend/src/optimizer/plan_node/stream_dedup.rs b/src/frontend/src/optimizer/plan_node/stream_dedup.rs index 44acf722eae6b..847616629355e 100644 --- a/src/frontend/src/optimizer/plan_node/stream_dedup.rs +++ b/src/frontend/src/optimizer/plan_node/stream_dedup.rs @@ -18,9 +18,9 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody; use risingwave_pb::stream_plan::DedupNode; use super::generic::{self, GenericPlanNode, GenericPlanRef}; +use super::stream::StreamPlanRef; use super::utils::{impl_distill_by_unit, TableCatalogBuilder}; use super::{ExprRewritable, PlanBase, PlanTreeNodeUnary, StreamNode}; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::PlanRef; use crate::stream_fragmenter::BuildFragmentGraphState; use crate::TableCatalog; diff --git a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs index ede1ae68b0825..580c3563bfa09 100644 --- a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs @@ -24,7 +24,6 @@ use super::generic::{self}; use super::utils::{childless_record, Distill}; use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, StreamNode}; use crate::expr::{Expr, ExprRewriter}; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::utils::IndicesDisplay; use crate::optimizer::plan_node::{EqJoinPredicate, EqJoinPredicateDisplay}; use crate::optimizer::property::Distribution; diff --git a/src/frontend/src/optimizer/plan_node/stream_derive.rs b/src/frontend/src/optimizer/plan_node/stream_derive.rs deleted file mode 100644 index f3da2b1b6a1dd..0000000000000 --- a/src/frontend/src/optimizer/plan_node/stream_derive.rs +++ /dev/null @@ -1,630 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use risingwave_common::catalog::Schema; - -use super::generic::GenericPlanNode; -use super::stream::*; -use crate::optimizer::optimizer_context::OptimizerContextRef; -use crate::optimizer::property::{Distribution, FunctionalDependencySet}; -use crate::utils::ColIndexMappingRewriteExt; - -impl GenericPlanNode for DynamicFilter { - fn schema(&self) -> Schema { - todo!("new plan node derivation") - } - - fn stream_key(&self) -> Option> { - todo!("new plan node derivation") - } - - fn ctx(&self) -> OptimizerContextRef { - todo!("new plan node derivation") - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - todo!("new plan node derivation") - } -} -impl StreamPlanNode for DynamicFilter { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for Exchange { - fn schema(&self) -> Schema { - todo!("new plan node derivation") - } - - fn stream_key(&self) -> Option> { - todo!("new plan node derivation") - } - - fn ctx(&self) -> OptimizerContextRef { - todo!("new plan node derivation") - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - todo!("new plan node derivation") - } -} - -impl StreamPlanNode for Exchange { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for DeltaJoin { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for DeltaJoin { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for Expand { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for Expand { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for Filter { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for Filter { - fn distribution(&self) -> Distribution { - self.core.input.distribution().clone() - } - - fn append_only(&self) -> bool { - self.core.input.append_only() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for SimpleAgg { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for SimpleAgg { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for GroupTopN { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for GroupTopN { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for HashAgg { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for HashAgg { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for HashJoin { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for HashJoin { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for HopWindow { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for HopWindow { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for IndexScan { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for IndexScan { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for StatelessSimpleAgg { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for StatelessSimpleAgg { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for Materialize { - fn schema(&self) -> Schema { - todo!("new plan node derivation") - } - - fn stream_key(&self) -> Option> { - todo!("new plan node derivation") - } - - fn ctx(&self) -> OptimizerContextRef { - todo!("new plan node derivation") - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - todo!("new plan node derivation") - } -} - -impl StreamPlanNode for Materialize { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for ProjectSet { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for ProjectSet { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for Project { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for Project { - fn distribution(&self) -> Distribution { - self.core - .i2o_col_mapping() - .rewrite_provided_distribution(self.core.input.distribution()) - } - - fn append_only(&self) -> bool { - self.core.input.append_only() - } - - fn emit_on_window_close(&self) -> bool { - self.core.input.emit_on_window_close() - } -} - -impl GenericPlanNode for Sink { - fn schema(&self) -> Schema { - todo!("new plan node derivation") - } - - fn stream_key(&self) -> Option> { - todo!("new plan node derivation") - } - - fn ctx(&self) -> OptimizerContextRef { - todo!("new plan node derivation") - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - todo!("new plan node derivation") - } -} - -impl StreamPlanNode for Sink { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for Source { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for Source { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for TableScan { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for TableScan { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} - -impl GenericPlanNode for TopN { - fn schema(&self) -> Schema { - self.core.schema() - } - - fn stream_key(&self) -> Option> { - self.core.stream_key() - } - - fn ctx(&self) -> OptimizerContextRef { - self.core.ctx() - } - - fn functional_dependency(&self) -> FunctionalDependencySet { - self.core.functional_dependency() - } -} - -impl StreamPlanNode for TopN { - fn distribution(&self) -> Distribution { - todo!() - } - - fn append_only(&self) -> bool { - todo!() - } - - fn emit_on_window_close(&self) -> bool { - todo!() - } -} diff --git a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs index 427c0de4c89c1..797091338915e 100644 --- a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs +++ b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs @@ -21,7 +21,6 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody; use super::generic::{self, PlanWindowFunction}; use super::utils::{impl_distill_by_unit, TableCatalogBuilder}; use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::stream_fragmenter::BuildFragmentGraphState; use crate::TableCatalog; diff --git a/src/frontend/src/optimizer/plan_node/stream_exchange.rs b/src/frontend/src/optimizer/plan_node/stream_exchange.rs index 45bcccff7a6cf..0fa1713bf4488 100644 --- a/src/frontend/src/optimizer/plan_node/stream_exchange.rs +++ b/src/frontend/src/optimizer/plan_node/stream_exchange.rs @@ -16,7 +16,6 @@ use pretty_xmlish::{Pretty, XmlNode}; use risingwave_pb::stream_plan::stream_node::NodeBody; use risingwave_pb::stream_plan::{DispatchStrategy, DispatcherType, ExchangeNode}; -use super::stream::StreamPlanRef; use super::utils::{childless_record, plan_node_name, Distill}; use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::optimizer::property::{Distribution, DistributionDisplay}; diff --git a/src/frontend/src/optimizer/plan_node/stream_expand.rs b/src/frontend/src/optimizer/plan_node/stream_expand.rs index 619bc98fc6406..c7a59b1f847fe 100644 --- a/src/frontend/src/optimizer/plan_node/stream_expand.rs +++ b/src/frontend/src/optimizer/plan_node/stream_expand.rs @@ -17,7 +17,6 @@ use risingwave_pb::stream_plan::expand_node::Subset; use risingwave_pb::stream_plan::stream_node::PbNodeBody; use risingwave_pb::stream_plan::ExpandNode; -use super::stream::StreamPlanRef; use super::utils::impl_distill_by_unit; use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::optimizer::property::Distribution; diff --git a/src/frontend/src/optimizer/plan_node/stream_filter.rs b/src/frontend/src/optimizer/plan_node/stream_filter.rs index 285a8086f5c81..ed4d506b47aef 100644 --- a/src/frontend/src/optimizer/plan_node/stream_filter.rs +++ b/src/frontend/src/optimizer/plan_node/stream_filter.rs @@ -15,7 +15,6 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody; use risingwave_pb::stream_plan::FilterNode; -use super::stream::StreamPlanRef; use super::utils::impl_distill_by_unit; use super::{generic, ExprRewritable, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::expr::{Expr, ExprImpl, ExprRewriter}; diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs index ea0925cf33bb7..989de9f8757e2 100644 --- a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs @@ -20,6 +20,7 @@ use risingwave_pb::plan_common::JoinType; use risingwave_pb::stream_plan::stream_node::NodeBody; use risingwave_pb::stream_plan::{DeltaExpression, HashJoinNode, PbInequalityPair}; +use super::generic::Join; use super::utils::{childless_record, plan_node_name, watermark_pretty, Distill}; use super::{ generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, StreamDeltaJoin, StreamNode, @@ -366,15 +367,14 @@ impl StreamNode for StreamHashJoin { let dk_indices_in_jk = self.derive_dist_key_in_join_key(); - use super::stream::HashJoin; let (left_table, left_degree_table, left_deduped_input_pk_indices) = - HashJoin::infer_internal_and_degree_table_catalog( + Join::infer_internal_and_degree_table_catalog( self.left().plan_base(), left_jk_indices, dk_indices_in_jk.clone(), ); let (right_table, right_degree_table, right_deduped_input_pk_indices) = - HashJoin::infer_internal_and_degree_table_catalog( + Join::infer_internal_and_degree_table_catalog( self.right().plan_base(), right_jk_indices, dk_indices_in_jk, diff --git a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs index ec18ee2588770..3780a6cda3f57 100644 --- a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs +++ b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs @@ -17,7 +17,6 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_pb::stream_plan::stream_node::PbNodeBody; use risingwave_pb::stream_plan::HopWindowNode; -use super::stream::StreamPlanRef; use super::utils::{childless_record, watermark_pretty, Distill}; use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::expr::{Expr, ExprImpl, ExprRewriter}; diff --git a/src/frontend/src/optimizer/plan_node/stream_project.rs b/src/frontend/src/optimizer/plan_node/stream_project.rs index 3a159a957af47..8e1b30eaafad2 100644 --- a/src/frontend/src/optimizer/plan_node/stream_project.rs +++ b/src/frontend/src/optimizer/plan_node/stream_project.rs @@ -17,7 +17,6 @@ use pretty_xmlish::XmlNode; use risingwave_pb::stream_plan::stream_node::PbNodeBody; use risingwave_pb::stream_plan::ProjectNode; -use super::stream::StreamPlanRef; use super::utils::{childless_record, watermark_pretty, Distill}; use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::expr::{try_derive_watermark, Expr, ExprImpl, ExprRewriter, WatermarkDerivation}; diff --git a/src/frontend/src/optimizer/plan_node/stream_project_set.rs b/src/frontend/src/optimizer/plan_node/stream_project_set.rs index 619fec1f80d15..97c4b70433cb1 100644 --- a/src/frontend/src/optimizer/plan_node/stream_project_set.rs +++ b/src/frontend/src/optimizer/plan_node/stream_project_set.rs @@ -17,7 +17,6 @@ use itertools::Itertools; use risingwave_pb::stream_plan::stream_node::PbNodeBody; use risingwave_pb::stream_plan::ProjectSetNode; -use super::stream::StreamPlanRef; use super::utils::impl_distill_by_unit; use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::expr::{try_derive_watermark, ExprRewriter, WatermarkDerivation}; diff --git a/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs b/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs index c85bd10dc1632..083cb877cd4d6 100644 --- a/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs +++ b/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs @@ -17,7 +17,6 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody; use super::utils::{childless_record, Distill}; use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::property::Distribution; use crate::stream_fragmenter::BuildFragmentGraphState; diff --git a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs index 639b6c5782bb5..296c58944fac0 100644 --- a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs @@ -20,7 +20,6 @@ use super::generic::{self, PlanAggCall}; use super::utils::impl_distill_by_unit; use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode}; use crate::expr::ExprRewriter; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::property::RequiredDist; use crate::stream_fragmenter::BuildFragmentGraphState; diff --git a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs index f9fb325b8af8b..eb883103fe511 100644 --- a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs @@ -22,7 +22,6 @@ use super::utils::{childless_record, watermark_pretty, Distill}; use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, StreamNode}; use crate::expr::{Expr, ExprRewriter}; use crate::optimizer::plan_node::plan_tree_node::PlanTreeNodeUnary; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::utils::IndicesDisplay; use crate::optimizer::plan_node::{ EqJoinPredicate, EqJoinPredicateDisplay, StreamExchange, StreamTableScan, diff --git a/src/frontend/src/optimizer/plan_node/stream_union.rs b/src/frontend/src/optimizer/plan_node/stream_union.rs index 74e8dbcf5c48e..1d259115b5ced 100644 --- a/src/frontend/src/optimizer/plan_node/stream_union.rs +++ b/src/frontend/src/optimizer/plan_node/stream_union.rs @@ -22,7 +22,6 @@ use risingwave_pb::stream_plan::UnionNode; use super::utils::{childless_record, watermark_pretty, Distill}; use super::{generic, ExprRewritable, PlanRef}; use crate::optimizer::plan_node::generic::GenericPlanNode; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::plan_node::{PlanBase, PlanTreeNode, StreamNode}; use crate::stream_fragmenter::BuildFragmentGraphState; diff --git a/src/frontend/src/optimizer/property/distribution.rs b/src/frontend/src/optimizer/property/distribution.rs index c4b09bd910c5f..4fcaf959eac87 100644 --- a/src/frontend/src/optimizer/property/distribution.rs +++ b/src/frontend/src/optimizer/property/distribution.rs @@ -59,7 +59,6 @@ use risingwave_pb::batch_plan::ExchangeInfo; use super::super::plan_node::*; use crate::catalog::catalog_service::CatalogReader; use crate::catalog::FragmentId; -use crate::optimizer::plan_node::stream::StreamPlanRef; use crate::optimizer::property::Order; use crate::optimizer::PlanRef; use crate::scheduler::worker_node_manager::WorkerNodeSelector; @@ -314,23 +313,6 @@ impl RequiredDist { } } - #[allow(dead_code)] - pub fn enforce_stream_if_not_satisfies( - &self, - plan: stream::PlanRef, - ) -> Result { - if !plan.distribution().satisfies(self) { - // FIXME(st1page); - Ok(stream::Exchange { - dist: self.to_dist(), - input: plan, - } - .into()) - } else { - Ok(plan) - } - } - /// check if the distribution satisfies other required distribution pub fn satisfies(&self, required: &RequiredDist) -> bool { match self { From ccef1c58d650141f8b7e7821df32070cb4ab4a7b Mon Sep 17 00:00:00 2001 From: xiangjinwu <17769960+xiangjinwu@users.noreply.github.com> Date: Sat, 21 Oct 2023 15:19:22 +0800 Subject: [PATCH 14/16] feat(sink): enable kafka sink with `format plain encode protobuf` (#12858) --- ci/scripts/e2e-kafka-sink-test.sh | 7 ++ ci/workflows/pull-request.yml | 2 +- e2e_test/sink/kafka/protobuf.slt | 97 +++++++++++++++++++++ src/connector/src/lib.rs | 1 + src/connector/src/parser/protobuf/parser.rs | 2 +- src/connector/src/schema/mod.rs | 21 +++++ src/connector/src/schema/protobuf.rs | 57 ++++++++++++ src/connector/src/sink/formatter/mod.rs | 48 +++++++--- src/connector/src/sink/kafka.rs | 12 ++- src/connector/src/sink/kinesis.rs | 12 ++- src/connector/src/sink/pulsar.rs | 12 ++- src/frontend/src/handler/create_sink.rs | 2 +- 12 files changed, 256 insertions(+), 17 deletions(-) create mode 100644 e2e_test/sink/kafka/protobuf.slt create mode 100644 src/connector/src/schema/mod.rs create mode 100644 src/connector/src/schema/protobuf.rs diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh index 1e01903ec2042..06ef185f46e8b 100755 --- a/ci/scripts/e2e-kafka-sink-test.sh +++ b/ci/scripts/e2e-kafka-sink-test.sh @@ -131,3 +131,10 @@ sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/drop_sink.slt' ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --delete > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --delete > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --delete > /dev/null 2>&1 + +# test different encoding +echo "testing protobuf" +cp src/connector/src/test_data/proto_recursive/recursive.pb ./proto-recursive +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only-protobuf --create > /dev/null 2>&1 +sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/protobuf.slt' +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only-protobuf --delete > /dev/null 2>&1 diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index 364455d045668..ee74834dd5163 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -82,7 +82,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 + timeout_in_minutes: 17 retry: *auto-retry - label: "end-to-end test (parallel)" diff --git a/e2e_test/sink/kafka/protobuf.slt b/e2e_test/sink/kafka/protobuf.slt new file mode 100644 index 0000000000000..f69c4a9d07110 --- /dev/null +++ b/e2e_test/sink/kafka/protobuf.slt @@ -0,0 +1,97 @@ +statement ok +create table from_kafka with ( + connector = 'kafka', + topic = 'test-rw-sink-append-only-protobuf', + properties.bootstrap.server = '127.0.0.1:29092') +format plain encode protobuf ( + schema.location = 'file:///risingwave/proto-recursive', + message = 'recursive.AllTypes'); + +statement ok +create table into_kafka ( + bool_field bool, + string_field varchar, + bytes_field bytea, + float_field real, + double_field double precision, + int32_field int, + int64_field bigint, + sint32_field int, + sint64_field bigint, + sfixed32_field int, + sfixed64_field bigint, + nested_message_field struct, + repeated_int_field int[], + timestamp_field timestamptz, + oneof_int32 int); + +statement ok +insert into into_kafka values + (true, 'Rising', 'a0', 3.5, 4.25, 22, 23, 24, null, 26, 27, row(1, ''), array[4, 0, 4], '2006-01-02 15:04:05-07:00', 42), + (false, 'Wave', 'ZDF', 1.5, null, 11, 12, 13, 14, 15, 16, row(4, 'foo'), null, null, null); + +statement ok +flush; + +statement ok +create sink sink0 from into_kafka with ( + connector = 'kafka', + topic = 'test-rw-sink-append-only-protobuf', + properties.bootstrap.server = '127.0.0.1:29092') +format plain encode protobuf ( + force_append_only = true, + schema.location = 'file:///risingwave/proto-recursive', + message = 'recursive.AllTypes'); + +sleep 2s + +query TTTRRIIIIIITTTI +select + bool_field, + string_field, + bytes_field, + float_field, + double_field, + int32_field, + int64_field, + sint32_field, + sint64_field, + sfixed32_field, + sfixed64_field, + nested_message_field, + repeated_int_field, + timestamp_field, + oneof_int32 from from_kafka; +---- +t Rising \x6130 3.5 4.25 22 23 24 0 26 27 (1,) {4,0,4} (1136239445,0) 42 +f Wave \x5a4446 1.5 0 11 12 13 14 15 16 (4,foo) {} (0,0) 0 + +statement error failed to read file +create sink sink_err from into_kafka with ( + connector = 'kafka', + topic = 'test-rw-sink-append-only-protobuf', + properties.bootstrap.server = '127.0.0.1:29092') +format plain encode protobuf ( + force_append_only = true, + schema.location = 'file:///risingwave/proto-recursiv', + message = 'recursive.AllTypes'); + +statement error encode extra_column error: field not in proto +create sink sink_err as select 1 as extra_column with ( + connector = 'kafka', + topic = 'test-rw-sink-append-only-protobuf', + properties.bootstrap.server = '127.0.0.1:29092') +format plain encode protobuf ( + force_append_only = true, + schema.location = 'file:///risingwave/proto-recursive', + message = 'recursive.AllTypes'); + +statement error s3 URL not supported yet +create sink sink_err from into_kafka with ( + connector = 'kafka', + topic = 'test-rw-sink-append-only-protobuf', + properties.bootstrap.server = '127.0.0.1:29092') +format plain encode protobuf ( + force_append_only = true, + schema.location = 's3:///risingwave/proto-recursive', + message = 'recursive.AllTypes'); diff --git a/src/connector/src/lib.rs b/src/connector/src/lib.rs index 8201e10731a28..8ccf62486ce65 100644 --- a/src/connector/src/lib.rs +++ b/src/connector/src/lib.rs @@ -46,6 +46,7 @@ pub mod error; mod macros; pub mod parser; +pub mod schema; pub mod sink; pub mod source; diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs index b07a7b214d2c9..cc8dc0734bdfe 100644 --- a/src/connector/src/parser/protobuf/parser.rs +++ b/src/connector/src/parser/protobuf/parser.rs @@ -73,7 +73,7 @@ impl ProtobufAccessBuilder { #[derive(Debug, Clone)] pub struct ProtobufParserConfig { confluent_wire_type: bool, - message_descriptor: MessageDescriptor, + pub(crate) message_descriptor: MessageDescriptor, } impl ProtobufParserConfig { diff --git a/src/connector/src/schema/mod.rs b/src/connector/src/schema/mod.rs new file mode 100644 index 0000000000000..fff2576389cfd --- /dev/null +++ b/src/connector/src/schema/mod.rs @@ -0,0 +1,21 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod protobuf; + +const MESSAGE_NAME_KEY: &str = "message"; +const SCHEMA_LOCATION_KEY: &str = "schema.location"; + +#[derive(Debug)] +pub struct SchemaFetchError(pub String); diff --git a/src/connector/src/schema/protobuf.rs b/src/connector/src/schema/protobuf.rs new file mode 100644 index 0000000000000..d312f3d0d73eb --- /dev/null +++ b/src/connector/src/schema/protobuf.rs @@ -0,0 +1,57 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; + +use prost_reflect::MessageDescriptor; + +use super::{SchemaFetchError, MESSAGE_NAME_KEY, SCHEMA_LOCATION_KEY}; +use crate::aws_auth::AwsAuthProps; +use crate::parser::{EncodingProperties, ProtobufParserConfig, ProtobufProperties}; + +/// `aws_auth_props` is only required when reading `s3://` URL. +pub async fn fetch_descriptor( + format_options: &BTreeMap, + aws_auth_props: Option<&AwsAuthProps>, +) -> Result { + let row_schema_location = format_options + .get(SCHEMA_LOCATION_KEY) + .ok_or_else(|| SchemaFetchError(format!("{SCHEMA_LOCATION_KEY} required")))? + .clone(); + let message_name = format_options + .get(MESSAGE_NAME_KEY) + .ok_or_else(|| SchemaFetchError(format!("{MESSAGE_NAME_KEY} required")))? + .clone(); + + if row_schema_location.starts_with("s3") && aws_auth_props.is_none() { + return Err(SchemaFetchError("s3 URL not supported yet".into())); + } + + let enc = EncodingProperties::Protobuf(ProtobufProperties { + use_schema_registry: false, + row_schema_location, + message_name, + aws_auth_props: aws_auth_props.cloned(), + // name_strategy, topic, key_message_name, enable_upsert, client_config + ..Default::default() + }); + // Ideally, we should extract the schema loading logic from source parser to this place, + // and call this in both source and sink. + // But right now this function calls into source parser for its schema loading functionality. + // This reversed dependency will be fixed when we support schema registry. + let conf = ProtobufParserConfig::new(enc) + .await + .map_err(|e| SchemaFetchError(e.to_string()))?; + Ok(conf.message_descriptor) +} diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 44c8ed6f3ea29..5a1c7379795d5 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -28,7 +28,7 @@ pub use upsert::UpsertFormatter; use super::catalog::{SinkEncode, SinkFormat, SinkFormatDesc}; use super::encoder::KafkaConnectParams; -use crate::sink::encoder::{JsonEncoder, TimestampHandlingMode}; +use crate::sink::encoder::{JsonEncoder, ProtoEncoder, TimestampHandlingMode}; /// Transforms a `StreamChunk` into a sequence of key-value pairs according a specific format, /// for example append-only, upsert or debezium. @@ -61,27 +61,28 @@ macro_rules! tri { }; } -#[expect(clippy::enum_variant_names)] pub enum SinkFormatterImpl { AppendOnlyJson(AppendOnlyFormatter), + AppendOnlyProto(AppendOnlyFormatter), UpsertJson(UpsertFormatter), DebeziumJson(DebeziumJsonFormatter), } impl SinkFormatterImpl { - pub fn new( + pub async fn new( format_desc: &SinkFormatDesc, schema: Schema, pk_indices: Vec, db_name: String, sink_from_name: String, ) -> Result { - if format_desc.encode != SinkEncode::Json { - return Err(SinkError::Config(anyhow!( - "sink encode unsupported: {:?}", + let err_unsupported = || { + Err(SinkError::Config(anyhow!( + "sink format/encode unsupported: {:?} {:?}", + format_desc.format, format_desc.encode, - ))); - } + ))) + }; match format_desc.format { SinkFormat::AppendOnly => { @@ -92,12 +93,32 @@ impl SinkFormatterImpl { TimestampHandlingMode::Milli, ) }); - let val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); - let formatter = AppendOnlyFormatter::new(key_encoder, val_encoder); - Ok(SinkFormatterImpl::AppendOnlyJson(formatter)) + match format_desc.encode { + SinkEncode::Json => { + let val_encoder = + JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); + let formatter = AppendOnlyFormatter::new(key_encoder, val_encoder); + Ok(SinkFormatterImpl::AppendOnlyJson(formatter)) + } + SinkEncode::Protobuf => { + // By passing `None` as `aws_auth_props`, reading from `s3://` not supported yet. + let descriptor = + crate::schema::protobuf::fetch_descriptor(&format_desc.options, None) + .await + .map_err(|e| SinkError::Config(anyhow!("{e:?}")))?; + let val_encoder = ProtoEncoder::new(schema, None, descriptor)?; + let formatter = AppendOnlyFormatter::new(key_encoder, val_encoder); + Ok(SinkFormatterImpl::AppendOnlyProto(formatter)) + } + SinkEncode::Avro => err_unsupported(), + } } SinkFormat::Debezium => { + if format_desc.encode != SinkEncode::Json { + return err_unsupported(); + } + Ok(SinkFormatterImpl::DebeziumJson(DebeziumJsonFormatter::new( schema, pk_indices, @@ -107,6 +128,10 @@ impl SinkFormatterImpl { ))) } SinkFormat::Upsert => { + if format_desc.encode != SinkEncode::Json { + return err_unsupported(); + } + let mut key_encoder = JsonEncoder::new( schema.clone(), Some(pk_indices), @@ -146,6 +171,7 @@ macro_rules! dispatch_sink_formatter_impl { ($impl:expr, $name:ident, $body:expr) => { match $impl { SinkFormatterImpl::AppendOnlyJson($name) => $body, + SinkFormatterImpl::AppendOnlyProto($name) => $body, SinkFormatterImpl::UpsertJson($name) => $body, SinkFormatterImpl::DebeziumJson($name) => $body, } diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs index ef4efe88e43f8..a204a8d121706 100644 --- a/src/connector/src/sink/kafka.rs +++ b/src/connector/src/sink/kafka.rs @@ -314,7 +314,8 @@ impl Sink for KafkaSink { self.pk_indices.clone(), self.db_name.clone(), self.sink_from_name.clone(), - )?; + ) + .await?; KafkaLogSinker::new(self.config.clone(), formatter).await } @@ -326,6 +327,15 @@ impl Sink for KafkaSink { self.format_desc.format ))); } + // Check for formatter constructor error, before it is too late for error reporting. + SinkFormatterImpl::new( + &self.format_desc, + self.schema.clone(), + self.pk_indices.clone(), + self.db_name.clone(), + self.sink_from_name.clone(), + ) + .await?; // Try Kafka connection. // There is no such interface for kafka producer to validate a connection diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs index 741bc1bb1db14..dd8518af39948 100644 --- a/src/connector/src/sink/kinesis.rs +++ b/src/connector/src/sink/kinesis.rs @@ -79,6 +79,15 @@ impl Sink for KinesisSink { "kinesis sink requires partition key (please define in `primary_key` field)", ))); } + // Check for formatter constructor error, before it is too late for error reporting. + SinkFormatterImpl::new( + &self.format_desc, + self.schema.clone(), + self.pk_indices.clone(), + self.db_name.clone(), + self.sink_from_name.clone(), + ) + .await?; // check reachability let client = self.config.common.build_client().await?; @@ -145,7 +154,8 @@ impl KinesisSinkWriter { sink_from_name: String, ) -> Result { let formatter = - SinkFormatterImpl::new(format_desc, schema, pk_indices, db_name, sink_from_name)?; + SinkFormatterImpl::new(format_desc, schema, pk_indices, db_name, sink_from_name) + .await?; let client = config .common .build_client() diff --git a/src/connector/src/sink/pulsar.rs b/src/connector/src/sink/pulsar.rs index 16514da4c4d23..f980b2ad9f9b1 100644 --- a/src/connector/src/sink/pulsar.rs +++ b/src/connector/src/sink/pulsar.rs @@ -180,6 +180,15 @@ impl Sink for PulsarSink { self.format_desc.format ))); } + // Check for formatter constructor error, before it is too late for error reporting. + SinkFormatterImpl::new( + &self.format_desc, + self.schema.clone(), + self.downstream_pk.clone(), + self.db_name.clone(), + self.sink_from_name.clone(), + ) + .await?; // Validate pulsar connection. let pulsar = self.config.common.build_client().await?; @@ -211,7 +220,8 @@ impl PulsarSinkWriter { sink_from_name: String, ) -> Result { let formatter = - SinkFormatterImpl::new(format_desc, schema, downstream_pk, db_name, sink_from_name)?; + SinkFormatterImpl::new(format_desc, schema, downstream_pk, db_name, sink_from_name) + .await?; let pulsar = config.common.build_client().await?; let producer = build_pulsar_producer(&pulsar, &config).await?; Ok(Self { diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs index 80335d278c8d4..32279dd4e70eb 100644 --- a/src/frontend/src/handler/create_sink.rs +++ b/src/frontend/src/handler/create_sink.rs @@ -266,7 +266,7 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock hashmap!( - Format::Plain => vec![Encode::Json], + Format::Plain => vec![Encode::Json, Encode::Protobuf], Format::Upsert => vec![Encode::Json], Format::Debezium => vec![Encode::Json], ), From 6833305cce8c9a1d23573f1972bf8dfa0ffaee7a Mon Sep 17 00:00:00 2001 From: Noel Kwan <47273164+kwannoel@users.noreply.github.com> Date: Sat, 21 Oct 2023 15:23:57 +0800 Subject: [PATCH 15/16] feat(meta): recover stream job progress (#12167) Co-authored-by: August --- Makefile.toml | 1 - ci/scripts/deterministic-recovery-test.sh | 10 +- ci/scripts/run-backfill-tests.sh | 379 +++++++++++++++++- ci/scripts/run-e2e-test.sh | 2 +- ci/scripts/sql/backfill/insert.sql | 2 +- ci/workflows/main-cron.yml | 2 +- ci/workflows/pull-request.yml | 4 +- .../background_ddl/common/create_bg_mv.slt | 5 + .../background_ddl/common/create_fg_mv.slt | 2 + .../background_ddl/common/create_index.slt | 2 + .../background_ddl/common/create_sink.slt | 2 + .../background_ddl/common/create_table.slt | 8 + e2e_test/background_ddl/common/drop_index.slt | 2 + e2e_test/background_ddl/common/drop_mv.slt | 2 + e2e_test/background_ddl/common/drop_sink.slt | 2 + e2e_test/background_ddl/common/drop_table.slt | 2 + .../common/validate_backfilled_mv.slt | 4 + .../common/validate_no_jobs.slt | 4 + .../common/validate_one_job.slt | 4 + e2e_test/background_ddl/sim.slt | 1 + e2e_test/background_ddl/sim/basic.slt | 63 +++ src/frontend/src/catalog/table_catalog.rs | 40 +- .../optimizer/plan_node/stream_materialize.rs | 3 +- src/frontend/src/optimizer/plan_node/utils.rs | 5 +- src/meta/service/src/notification_service.rs | 13 +- src/meta/src/barrier/command.rs | 77 ++-- src/meta/src/barrier/mod.rs | 82 ++-- src/meta/src/barrier/notifier.rs | 2 +- src/meta/src/barrier/progress.rs | 204 ++++++++-- src/meta/src/barrier/recovery.rs | 138 ++++++- src/meta/src/error.rs | 11 + src/meta/src/manager/catalog/database.rs | 36 +- src/meta/src/manager/catalog/fragment.rs | 92 ++++- src/meta/src/manager/catalog/mod.rs | 277 +++++++++++-- src/meta/src/manager/streaming_job.rs | 11 +- src/meta/src/model/stream.rs | 15 +- src/meta/src/rpc/ddl_controller.rs | 76 +++- src/meta/src/stream/stream_manager.rs | 75 +++- .../recovery/background_ddl.rs | 96 +++++ .../tests/integration_tests/recovery/mod.rs | 1 + 40 files changed, 1546 insertions(+), 211 deletions(-) create mode 100644 e2e_test/background_ddl/common/create_bg_mv.slt create mode 100644 e2e_test/background_ddl/common/create_fg_mv.slt create mode 100644 e2e_test/background_ddl/common/create_index.slt create mode 100644 e2e_test/background_ddl/common/create_sink.slt create mode 100644 e2e_test/background_ddl/common/create_table.slt create mode 100644 e2e_test/background_ddl/common/drop_index.slt create mode 100644 e2e_test/background_ddl/common/drop_mv.slt create mode 100644 e2e_test/background_ddl/common/drop_sink.slt create mode 100644 e2e_test/background_ddl/common/drop_table.slt create mode 100644 e2e_test/background_ddl/common/validate_backfilled_mv.slt create mode 100644 e2e_test/background_ddl/common/validate_no_jobs.slt create mode 100644 e2e_test/background_ddl/common/validate_one_job.slt create mode 100644 e2e_test/background_ddl/sim.slt create mode 100644 e2e_test/background_ddl/sim/basic.slt create mode 100644 src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs diff --git a/Makefile.toml b/Makefile.toml index 1203a847e1e94..86e9cea136bb5 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1317,7 +1317,6 @@ dependencies = ["k", "l", "check-logs", "wait-processes-exit"] description = "Kill cluster, dump logs and check logs" [tasks.wait-processes-exit] -private = true category = "Misc" description = "Wait for RisingWave processes to exit" script = """ diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 0d3a7b3fabed4..6514fe1f7c0c3 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -9,11 +9,19 @@ echo "--- Download artifacts" download-and-decompress-artifact risingwave_simulation . chmod +x ./risingwave_simulation -export RUST_LOG="info,risingwave_meta::barrier::recovery=debug" +export RUST_LOG="info,\ +risingwave_meta::barrier::recovery=debug,\ +risingwave_meta::rpc::ddl_controller=debug,\ +risingwave_meta::barrier::mod=debug,\ +risingwave_simulation=debug" export LOGDIR=.risingwave/log mkdir -p $LOGDIR +# FIXME(kwannoel): Why is this failing? +# echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl" +# seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/background_ddl/sim/basic.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' + echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, ddl" seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh index 86dbcb376c255..d0d5eafb3c917 100755 --- a/ci/scripts/run-backfill-tests.sh +++ b/ci/scripts/run-backfill-tests.sh @@ -1,22 +1,31 @@ #!/usr/bin/env bash # Runs backfill tests. -# NOTE(kwannoel): -# The following scenario is adapted in madsim's integration tests as well. -# But this script reproduces it more reliably (I'm not sure why.) -# Hence keeping it in case we ever need to debug backfill again. # USAGE: # ```sh # cargo make ci-start ci-backfill # ./ci/scripts/run-backfill-tests.sh # ``` +# Example progress: +# dev=> select * from rw_catalog.rw_ddl_progress; +# ddl_id | ddl_statement | progress | initialized_at +#--------+------------------------------------------------+----------+------------------------------- +# 1002 | CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t | 56.12% | 2023-09-27 06:37:06.636+00:00 +#(1 row) set -euo pipefail PARENT_PATH=$(dirname "${BASH_SOURCE[0]}") +TEST_DIR=$PWD/e2e_test +BACKGROUND_DDL_DIR=$TEST_DIR/background_ddl +COMMON_DIR=$BACKGROUND_DDL_DIR/common + +CLUSTER_PROFILE='ci-1cn-1fe-with-recovery' +export RUST_LOG="risingwave_meta=debug" + run_sql_file() { psql -h localhost -p 4566 -d dev -U root -f "$@" } @@ -29,40 +38,309 @@ flush() { run_sql "FLUSH;" } -basic() { - echo "--- e2e, test_backfill_basic" +cancel_stream_jobs() { + ID=$(run_sql "select ddl_id from rw_catalog.rw_ddl_progress;" | tail -3 | head -1 | grep -E -o "[0-9]*") + echo "CANCELLING STREAM_JOB: $ID" + run_sql "CANCEL JOBS $ID;" .risingwave/log/compute-node.log 2>&1 & +} + +# Test snapshot and upstream read. +test_snapshot_and_upstream_read() { + echo "--- e2e, ci-backfill, test_snapshot_and_upstream_read" cargo make ci-start ci-backfill + run_sql_file "$PARENT_PATH"/sql/backfill/create_base_table.sql # Provide snapshot run_sql_file "$PARENT_PATH"/sql/backfill/insert.sql + + # Provide updates ... run_sql_file "$PARENT_PATH"/sql/backfill/insert.sql & + + # ... and concurrently create mv. run_sql_file "$PARENT_PATH"/sql/backfill/create_mv.sql & wait run_sql_file "$PARENT_PATH"/sql/backfill/select.sql Self { + Self::Foreground + } +} + +impl CreateType { + fn from_prost(prost: PbCreateType) -> Self { + match prost { + PbCreateType::Background => Self::Background, + PbCreateType::Foreground => Self::Foreground, + PbCreateType::Unspecified => unreachable!(), + } + } + + pub(crate) fn to_prost(self) -> PbCreateType { + match self { + Self::Background => PbCreateType::Background, + Self::Foreground => PbCreateType::Foreground, + } + } } #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] @@ -402,7 +437,7 @@ impl TableCatalog { created_at_epoch: self.created_at_epoch.map(|epoch| epoch.0), cleaned_by_watermark: self.cleaned_by_watermark, stream_job_status: PbStreamJobStatus::Creating.into(), - create_type: PbCreateType::Foreground.into(), + create_type: self.create_type.to_prost().into(), } } @@ -455,6 +490,7 @@ impl From for TableCatalog { let id = tb.id; let tb_conflict_behavior = tb.handle_pk_conflict_behavior(); let table_type = tb.get_table_type().unwrap(); + let create_type = tb.get_create_type().unwrap_or(PbCreateType::Foreground); let associated_source_id = tb.optional_associated_source_id.map(|id| match id { OptionalAssociatedSourceId::AssociatedSourceId(id) => id, }); @@ -514,6 +550,7 @@ impl From for TableCatalog { created_at_epoch: tb.created_at_epoch.map(Epoch::from), initialized_at_epoch: tb.initialized_at_epoch.map(Epoch::from), cleaned_by_watermark: matches!(tb.cleaned_by_watermark, true), + create_type: CreateType::from_prost(create_type), } } } @@ -660,6 +697,7 @@ mod tests { created_at_epoch: None, initialized_at_epoch: None, cleaned_by_watermark: false, + create_type: CreateType::Foreground, } ); assert_eq!(table, TableCatalog::from(table.to_prost(0, 0))); diff --git a/src/frontend/src/optimizer/plan_node/stream_materialize.rs b/src/frontend/src/optimizer/plan_node/stream_materialize.rs index 128f1eeb24ed9..fb17537bc90e6 100644 --- a/src/frontend/src/optimizer/plan_node/stream_materialize.rs +++ b/src/frontend/src/optimizer/plan_node/stream_materialize.rs @@ -26,7 +26,7 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody; use super::derive::derive_columns; use super::utils::{childless_record, Distill}; use super::{reorganize_elements_id, ExprRewritable, PlanRef, PlanTreeNodeUnary, StreamNode}; -use crate::catalog::table_catalog::{TableCatalog, TableType, TableVersion}; +use crate::catalog::table_catalog::{CreateType, TableCatalog, TableType, TableVersion}; use crate::catalog::FragmentId; use crate::optimizer::plan_node::derive::derive_pk; use crate::optimizer::plan_node::{PlanBase, PlanNodeMeta}; @@ -232,6 +232,7 @@ impl StreamMaterialize { created_at_epoch: None, initialized_at_epoch: None, cleaned_by_watermark: false, + create_type: CreateType::Foreground, // Will be updated in the handler itself. }) } diff --git a/src/frontend/src/optimizer/plan_node/utils.rs b/src/frontend/src/optimizer/plan_node/utils.rs index 475c5c0e32eb1..f167d73c53a46 100644 --- a/src/frontend/src/optimizer/plan_node/utils.rs +++ b/src/frontend/src/optimizer/plan_node/utils.rs @@ -23,7 +23,7 @@ use risingwave_common::catalog::{ }; use risingwave_common::util::sort_util::{ColumnOrder, OrderType}; -use crate::catalog::table_catalog::TableType; +use crate::catalog::table_catalog::{CreateType, TableType}; use crate::catalog::{ColumnId, FragmentId, TableCatalog, TableId}; use crate::optimizer::property::Cardinality; use crate::utils::WithOptions; @@ -177,6 +177,9 @@ impl TableCatalogBuilder { created_at_epoch: None, initialized_at_epoch: None, cleaned_by_watermark: false, + // NOTE(kwannoel): This may not match the create type of the materialized table. + // It should be ignored for internal tables. + create_type: CreateType::Foreground, } } diff --git a/src/meta/service/src/notification_service.rs b/src/meta/service/src/notification_service.rs index 18bdc30c4165f..bd247c1e18980 100644 --- a/src/meta/service/src/notification_service.rs +++ b/src/meta/service/src/notification_service.rs @@ -14,7 +14,7 @@ use itertools::Itertools; use risingwave_pb::backup_service::MetaBackupManifestId; -use risingwave_pb::catalog::{PbStreamJobStatus, Table}; +use risingwave_pb::catalog::Table; use risingwave_pb::common::worker_node::State::Running; use risingwave_pb::common::{WorkerNode, WorkerType}; use risingwave_pb::hummock::WriteLimits; @@ -120,16 +120,7 @@ impl NotificationServiceImpl { async fn get_tables_and_creating_tables_snapshot(&self) -> (Vec, NotificationVersion) { let catalog_guard = self.catalog_manager.get_catalog_core_guard().await; - let mut tables = catalog_guard - .database - .list_tables() - .into_iter() - .filter(|t| { - t.stream_job_status == PbStreamJobStatus::Unspecified as i32 - || t.stream_job_status == PbStreamJobStatus::Created as i32 - }) - .collect_vec(); - tables.extend(catalog_guard.database.list_creating_tables()); + let tables = catalog_guard.database.list_tables(); let notification_version = self.env.notification_manager().current_version().await; (tables, notification_version) } diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs index 8d8076e56a233..bbe60c010b94b 100644 --- a/src/meta/src/barrier/command.rs +++ b/src/meta/src/barrier/command.rs @@ -36,7 +36,8 @@ use uuid::Uuid; use super::info::BarrierActorInfo; use super::trace::TracedEpoch; use crate::barrier::CommandChanges; -use crate::manager::{FragmentManagerRef, WorkerId}; +use crate::hummock::HummockManagerRef; +use crate::manager::{CatalogManagerRef, FragmentManagerRef, WorkerId}; use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments}; use crate::stream::{build_actor_connector_splits, SourceManagerRef, SplitAssignment}; use crate::MetaResult; @@ -216,7 +217,9 @@ impl Command { /// [`CommandContext`] is used for generating barrier and doing post stuffs according to the given /// [`Command`]. pub struct CommandContext { - fragment_manager: FragmentManagerRef, + pub fragment_manager: FragmentManagerRef, + catalog_manager: CatalogManagerRef, + hummock_manager: HummockManagerRef, client_pool: StreamClientPoolRef, @@ -247,6 +250,8 @@ impl CommandContext { #[allow(clippy::too_many_arguments)] pub(super) fn new( fragment_manager: FragmentManagerRef, + catalog_manager: CatalogManagerRef, + hummock_manager: HummockManagerRef, client_pool: StreamClientPoolRef, info: BarrierActorInfo, prev_epoch: TracedEpoch, @@ -259,6 +264,8 @@ impl CommandContext { ) -> Self { Self { fragment_manager, + catalog_manager, + hummock_manager, client_pool, info: Arc::new(info), prev_epoch, @@ -663,7 +670,51 @@ impl CommandContext { Command::CancelStreamingJob(table_fragments) => { let node_actors = table_fragments.worker_actor_ids(); self.clean_up(node_actors).await?; - // Drop fragment info in meta store. + + // NOTE(kwannoel): At this point, meta has already registered the table ids. + // We should unregister them. + // This is required for background ddl, for foreground ddl this is a no-op. + // Foreground ddl is handled entirely by stream manager, so it will unregister + // the table ids on failure. + // On the other hand background ddl could be handled by barrier manager. + // It won't clean the tables on failure, + // since the failure could be recoverable. + // As such it needs to be handled here. + let table_id = table_fragments.table_id().table_id; + let mut table_ids = table_fragments.internal_table_ids(); + table_ids.push(table_id); + if let Err(e) = self.hummock_manager.unregister_table_ids(&table_ids).await { + tracing::warn!("Failed to unregister compaction group for {:#?}. They will be cleaned up on node restart. {:#?}", &table_ids, e); + } + + // NOTE(kwannoel): At this point, catalog manager has persisted the tables already. + // We need to cleanup the table state. So we can do it here. + // The logic is the same as above, for hummock_manager.unregister_table_ids. + if let Err(e) = self + .catalog_manager + .cancel_create_table_procedure( + table_fragments.table_id().table_id, + table_fragments.internal_table_ids(), + ) + .await + { + let table_id = table_fragments.table_id().table_id; + tracing::warn!( + table_id, + reason=?e, + "cancel_create_table_procedure failed for CancelStreamingJob", + ); + // If failed, check that table is not in meta store. + // If any table is, just panic, let meta do bootstrap recovery. + // Otherwise our persisted state is dirty. + let mut table_ids = table_fragments.internal_table_ids(); + table_ids.push(table_id); + self.catalog_manager.assert_tables_deleted(table_ids).await; + } + + // We need to drop table fragments here, + // since this is not done in stream manager (foreground ddl) + // OR barrier manager (background ddl) self.fragment_manager .drop_table_fragments_vec(&HashSet::from_iter(std::iter::once( table_fragments.table_id(), @@ -791,24 +842,4 @@ impl CommandContext { Ok(()) } - - /// Do some stuffs before the barrier is `finish`ed. Only used for `CreateStreamingJob`. - pub async fn pre_finish(&self) -> MetaResult<()> { - #[allow(clippy::single_match)] - match &self.command { - Command::CreateStreamingJob { - table_fragments, .. - } => { - // Update the state of the table fragments from `Creating` to `Created`, so that the - // fragments can be scaled. - self.fragment_manager - .mark_table_fragments_created(table_fragments.table_id()) - .await?; - } - - _ => {} - } - - Ok(()) - } } diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index 8924992c2e18e..ed6ad289a5a68 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -50,7 +50,7 @@ use self::info::BarrierActorInfo; use self::notifier::Notifier; use self::progress::TrackingCommand; use crate::barrier::notifier::BarrierInfo; -use crate::barrier::progress::CreateMviewProgressTracker; +use crate::barrier::progress::{CreateMviewProgressTracker, TrackingJob}; use crate::barrier::BarrierEpochState::{Completed, InFlight}; use crate::hummock::HummockManagerRef; use crate::manager::sink_coordination::SinkCoordinatorManager; @@ -58,7 +58,7 @@ use crate::manager::{ CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, LocalNotification, MetaSrvEnv, WorkerId, }; -use crate::model::{ActorId, BarrierManagerState}; +use crate::model::{ActorId, BarrierManagerState, TableFragments}; use crate::rpc::metrics::MetaMetrics; use crate::stream::SourceManagerRef; use crate::{MetaError, MetaResult}; @@ -75,6 +75,35 @@ pub use self::command::{Command, Reschedule}; pub use self::schedule::BarrierScheduler; pub use self::trace::TracedEpoch; +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(crate) struct TableMap { + inner: HashMap, +} + +impl TableMap { + pub fn remove(&mut self, table_id: &TableId) -> Option { + self.inner.remove(table_id) + } +} + +impl From> for TableMap { + fn from(inner: HashMap) -> Self { + Self { inner } + } +} + +impl From> for HashMap { + fn from(table_map: TableMap) -> Self { + table_map.inner + } +} + +pub(crate) type TableActorMap = TableMap>; +pub(crate) type TableUpstreamMvCountMap = TableMap>; +pub(crate) type TableDefinitionMap = TableMap; +pub(crate) type TableNotifierMap = TableMap; +pub(crate) type TableFragmentMap = TableMap; + /// Status of barrier manager. enum BarrierManagerStatus { /// Barrier manager is starting. @@ -177,7 +206,7 @@ struct CheckpointControl { metrics: Arc, /// Get notified when we finished Create MV and collect a barrier(checkpoint = true) - finished_commands: Vec, + finished_commands: Vec, } impl CheckpointControl { @@ -194,8 +223,8 @@ impl CheckpointControl { } /// Stash a command to finish later. - fn stash_command_to_finish(&mut self, finished_command: TrackingCommand) { - self.finished_commands.push(finished_command); + fn stash_command_to_finish(&mut self, finished_job: TrackingJob) { + self.finished_commands.push(finished_job); } /// Finish stashed commands. If the current barrier is not a `checkpoint`, we will not finish @@ -205,31 +234,32 @@ impl CheckpointControl { async fn finish_commands(&mut self, checkpoint: bool) -> MetaResult { for command in self .finished_commands - .extract_if(|c| checkpoint || c.context.kind.is_barrier()) + .extract_if(|c| checkpoint || c.is_barrier()) { // The command is ready to finish. We can now call `pre_finish`. - command.context.pre_finish().await?; - command - .notifiers - .into_iter() - .for_each(Notifier::notify_finished); + command.pre_finish().await?; + command.notify_finished(); } Ok(!self.finished_commands.is_empty()) } - fn cancel_command(&mut self, cancelled_command: TrackingCommand) { - if let Some(index) = self.command_ctx_queue.iter().position(|x| { - x.command_ctx.prev_epoch.value() == cancelled_command.context.prev_epoch.value() - }) { - self.command_ctx_queue.remove(index); - self.remove_changes(cancelled_command.context.command.changes()); + fn cancel_command(&mut self, cancelled_job: TrackingJob) { + if let TrackingJob::New(cancelled_command) = cancelled_job { + if let Some(index) = self.command_ctx_queue.iter().position(|x| { + x.command_ctx.prev_epoch.value() == cancelled_command.context.prev_epoch.value() + }) { + self.command_ctx_queue.remove(index); + self.remove_changes(cancelled_command.context.command.changes()); + } + } else { + // Recovered jobs do not need to be cancelled since only `RUNNING` actors will get recovered. } } fn cancel_stashed_command(&mut self, id: TableId) { self.finished_commands - .retain(|x| x.context.table_to_create() != Some(id)); + .retain(|x| x.table_to_create() != Some(id)); } /// Before resolving the actors to be sent or collected, we should first record the newly @@ -596,7 +626,7 @@ impl GlobalBarrierManager { let paused = self.take_pause_on_bootstrap().await.unwrap_or(false); let paused_reason = paused.then_some(PausedReason::Manual); - self.recovery(prev_epoch, paused_reason) + self.recovery(prev_epoch, paused_reason, true) .instrument(span) .await }; @@ -693,6 +723,8 @@ impl GlobalBarrierManager { let command_ctx = Arc::new(CommandContext::new( self.fragment_manager.clone(), + self.catalog_manager.clone(), + self.hummock_manager.clone(), self.env.stream_client_pool_ref(), info, prev_epoch.clone(), @@ -910,6 +942,7 @@ impl GlobalBarrierManager { let fail_nodes = complete_nodes .drain(index..) .chain(checkpoint_control.barrier_failed().into_iter()); + tracing::warn!("Failed to commit epoch {}: {:?}", prev_epoch, err); self.failure_recovery(err, fail_nodes, state, checkpoint_control) .await; } @@ -937,11 +970,7 @@ impl GlobalBarrierManager { } if self.enable_recovery { - // If failed, enter recovery mode. self.set_status(BarrierManagerStatus::Recovering).await; - let mut tracker = self.tracker.lock().await; - *tracker = CreateMviewProgressTracker::new(); - let latest_snapshot = self.hummock_manager.latest_snapshot(); let prev_epoch = TracedEpoch::new(latest_snapshot.committed_epoch.into()); // we can only recovery from the committed epoch let span = tracing::info_span!( @@ -950,7 +979,12 @@ impl GlobalBarrierManager { prev_epoch = prev_epoch.value().0 ); - *state = self.recovery(prev_epoch, None).instrument(span).await; + // No need to clean dirty tables for barrier recovery, + // The foreground stream job should cleanup their own tables. + *state = self + .recovery(prev_epoch, None, false) + .instrument(span) + .await; self.set_status(BarrierManagerStatus::Running).await; } else { panic!("failed to execute barrier: {:?}", err); diff --git a/src/meta/src/barrier/notifier.rs b/src/meta/src/barrier/notifier.rs index 88acd9cd3dd7a..b28c5b01d53d9 100644 --- a/src/meta/src/barrier/notifier.rs +++ b/src/meta/src/barrier/notifier.rs @@ -30,7 +30,7 @@ pub struct BarrierInfo { /// Used for notifying the status of a scheduled command/barrier. #[derive(Debug, Default)] -pub(super) struct Notifier { +pub(crate) struct Notifier { /// Get notified when scheduled barrier is injected to compute nodes. pub injected: Option>, diff --git a/src/meta/src/barrier/progress.rs b/src/meta/src/barrier/progress.rs index 6344667130b60..22cd6f8d9e200 100644 --- a/src/meta/src/barrier/progress.rs +++ b/src/meta/src/barrier/progress.rs @@ -25,14 +25,18 @@ use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgres use super::command::CommandContext; use super::notifier::Notifier; -use crate::barrier::Command; -use crate::model::ActorId; +use crate::barrier::{ + Command, TableActorMap, TableDefinitionMap, TableFragmentMap, TableNotifierMap, + TableUpstreamMvCountMap, +}; +use crate::manager::{FragmentManager, FragmentManagerRef}; +use crate::model::{ActorId, TableFragments}; +use crate::MetaResult; -type CreateMviewEpoch = Epoch; type ConsumedRows = u64; #[derive(Clone, Copy, Debug)] -enum ChainState { +pub enum ChainState { Init, ConsumingUpstream(Epoch, ConsumedRows), Done(ConsumedRows), @@ -45,10 +49,9 @@ struct Progress { done_count: usize, - /// Creating mv id. - creating_mv_id: TableId, - - /// Upstream mv count. Keep track of how many times each upstream MV appears. + /// Upstream mv count. + /// Keep track of how many times each upstream MV + /// appears in this stream job. upstream_mv_count: HashMap, /// Upstream mvs total key count. @@ -65,7 +68,6 @@ impl Progress { /// Create a [`Progress`] for some creating mview, with all `actors` containing the chain nodes. fn new( actors: impl IntoIterator, - creating_mv_id: TableId, upstream_mv_count: HashMap, upstream_total_key_count: u64, definition: String, @@ -79,7 +81,6 @@ impl Progress { Self { states, done_count: 0, - creating_mv_id, upstream_mv_count, upstream_total_key_count, consumed_rows: 0, @@ -139,6 +140,80 @@ impl Progress { } } +/// There are 2 kinds of `TrackingJobs`: +/// 1. `New`. This refers to the "New" type of tracking job. +/// It is instantiated and managed by the stream manager. +/// On recovery, the stream manager will stop managing the job. +/// 2. `Recovered`. This refers to the "Recovered" type of tracking job. +/// On recovery, the barrier manager will recover and start managing the job. +pub enum TrackingJob { + New(TrackingCommand), + Recovered(RecoveredTrackingJob), +} + +impl TrackingJob { + fn fragment_manager(&self) -> &FragmentManager { + match self { + TrackingJob::New(command) => command.context.fragment_manager.as_ref(), + TrackingJob::Recovered(recovered) => recovered.fragment_manager.as_ref(), + } + } + + pub(crate) fn is_barrier(&self) -> bool { + match self { + TrackingJob::Recovered(_) => true, + TrackingJob::New(command) => command.context.kind.is_barrier(), + } + } + + pub(crate) async fn pre_finish(&self) -> MetaResult<()> { + let table_fragments = match &self { + TrackingJob::New(command) => match &command.context.command { + Command::CreateStreamingJob { + table_fragments, .. + } => Some(table_fragments), + _ => None, + }, + TrackingJob::Recovered(recovered) => Some(&recovered.fragments), + }; + // Update the state of the table fragments from `Creating` to `Created`, so that the + // fragments can be scaled. + if let Some(table_fragments) = table_fragments { + self.fragment_manager() + .mark_table_fragments_created(table_fragments.table_id()) + .await?; + } + Ok(()) + } + + pub(crate) fn notify_finished(self) { + match self { + TrackingJob::New(command) => { + command + .notifiers + .into_iter() + .for_each(Notifier::notify_finished); + } + TrackingJob::Recovered(recovered) => { + recovered.finished.notify_finished(); + } + } + } + + pub(crate) fn table_to_create(&self) -> Option { + match self { + TrackingJob::New(command) => command.context.table_to_create(), + TrackingJob::Recovered(recovered) => Some(recovered.fragments.table_id()), + } + } +} + +pub struct RecoveredTrackingJob { + pub fragments: TableFragments, + pub finished: Notifier, + pub fragment_manager: FragmentManagerRef, +} + /// The command tracking by the [`CreateMviewProgressTracker`]. pub(super) struct TrackingCommand { /// The context of the command. @@ -150,15 +225,80 @@ pub(super) struct TrackingCommand { /// Track the progress of all creating mviews. When creation is done, `notify_finished` will be /// called on registered notifiers. +/// +/// Tracking is done as follows: +/// 1. We identify a `StreamJob` by its `TableId` of its `Materialized` table. +/// 2. For each stream job, there are several actors which run its tasks. +/// 3. With `progress_map` we can use the ID of the `StreamJob` to view its progress. +/// 4. With `actor_map` we can use an actor's `ActorId` to find the ID of the `StreamJob`. pub(super) struct CreateMviewProgressTracker { - /// Progress of the create-mview DDL indicated by the epoch. - progress_map: HashMap, + /// Progress of the create-mview DDL indicated by the TableId. + progress_map: HashMap, /// Find the epoch of the create-mview DDL by the actor containing the chain node. - actor_map: HashMap, + actor_map: HashMap, } impl CreateMviewProgressTracker { + /// This step recovers state from the meta side: + /// 1. `Tables`. + /// 2. `TableFragments`. + /// + /// Other state are persisted by the `BackfillExecutor`, such as: + /// 1. `CreateMviewProgress`. + /// 2. `Backfill` position. + pub fn recover( + table_map: TableActorMap, + mut upstream_mv_counts: TableUpstreamMvCountMap, + mut definitions: TableDefinitionMap, + version_stats: HummockVersionStats, + mut finished_notifiers: TableNotifierMap, + mut table_fragment_map: TableFragmentMap, + fragment_manager: FragmentManagerRef, + ) -> Self { + let mut actor_map = HashMap::new(); + let mut progress_map = HashMap::new(); + let table_map: HashMap<_, Vec> = table_map.into(); + for (creating_table_id, actors) in table_map { + // 1. Recover `ChainState` in the tracker. + let mut states = HashMap::new(); + for actor in actors { + actor_map.insert(actor, creating_table_id); + states.insert(actor, ChainState::ConsumingUpstream(Epoch(0), 0)); + } + let upstream_mv_count = upstream_mv_counts.remove(&creating_table_id).unwrap(); + let upstream_total_key_count = upstream_mv_count + .iter() + .map(|(upstream_mv, count)| { + *count as u64 + * version_stats + .table_stats + .get(&upstream_mv.table_id) + .map_or(0, |stat| stat.total_key_count as u64) + }) + .sum(); + let definition = definitions.remove(&creating_table_id).unwrap(); + let progress = Progress { + states, + done_count: 0, // Fill only after first barrier pass + upstream_mv_count, + upstream_total_key_count, + consumed_rows: 0, // Fill only after first barrier pass + definition, + }; + let tracking_job = TrackingJob::Recovered(RecoveredTrackingJob { + fragments: table_fragment_map.remove(&creating_table_id).unwrap(), + finished: finished_notifiers.remove(&creating_table_id).unwrap(), + fragment_manager: fragment_manager.clone(), + }); + progress_map.insert(creating_table_id, (progress, tracking_job)); + } + Self { + progress_map, + actor_map, + } + } + pub fn new() -> Self { Self { progress_map: Default::default(), @@ -168,9 +308,9 @@ impl CreateMviewProgressTracker { pub fn gen_ddl_progress(&self) -> Vec { self.progress_map - .values() - .map(|(x, _)| DdlProgress { - id: x.creating_mv_id.table_id as u64, + .iter() + .map(|(table_id, (x, _))| DdlProgress { + id: table_id.table_id as u64, statement: x.definition.clone(), progress: format!("{:.2}%", x.calculate_progress() * 100.0), }) @@ -183,7 +323,7 @@ impl CreateMviewProgressTracker { pub fn find_cancelled_command( &mut self, actors_to_cancel: HashSet, - ) -> Option { + ) -> Option { let epochs = actors_to_cancel .into_iter() .map(|actor_id| self.actor_map.get(&actor_id)) @@ -205,16 +345,11 @@ impl CreateMviewProgressTracker { &mut self, command: TrackingCommand, version_stats: &HummockVersionStats, - ) -> Option { + ) -> Option { let actors = command.context.actors_to_track(); if actors.is_empty() { // The command can be finished immediately. - return Some(command); - } - - let ddl_epoch = command.context.curr_epoch.value(); - for &actor in &actors { - self.actor_map.insert(actor, ddl_epoch); + return Some(TrackingJob::New(command)); } let (creating_mv_id, upstream_mv_count, upstream_total_key_count, definition) = @@ -258,14 +393,19 @@ impl CreateMviewProgressTracker { unreachable!("Must be CreateStreamingJob."); }; + for &actor in &actors { + self.actor_map.insert(actor, creating_mv_id); + } + let progress = Progress::new( actors, - creating_mv_id, upstream_mv_count, upstream_total_key_count, definition, ); - let old = self.progress_map.insert(ddl_epoch, (progress, command)); + let old = self + .progress_map + .insert(creating_mv_id, (progress, TrackingJob::New(command))); assert!(old.is_none()); None } @@ -277,9 +417,9 @@ impl CreateMviewProgressTracker { &mut self, progress: &CreateMviewProgress, version_stats: &HummockVersionStats, - ) -> Option { + ) -> Option { let actor = progress.chain_actor_id; - let Some(epoch) = self.actor_map.get(&actor).copied() else { + let Some(table_id) = self.actor_map.get(&actor).copied() else { // On restart, backfill will ALWAYS notify CreateMviewProgressTracker, // even if backfill is finished on recovery. // This is because we don't know if only this actor is finished, @@ -299,7 +439,7 @@ impl CreateMviewProgressTracker { ChainState::ConsumingUpstream(progress.consumed_epoch.into(), progress.consumed_rows) }; - match self.progress_map.entry(epoch) { + match self.progress_map.entry(table_id) { Entry::Occupied(mut o) => { let progress = &mut o.get_mut().0; @@ -307,6 +447,7 @@ impl CreateMviewProgressTracker { .upstream_mv_count .iter() .map(|(upstream_mv, count)| { + assert_ne!(*count, 0); *count as u64 * version_stats .table_stats @@ -318,7 +459,10 @@ impl CreateMviewProgressTracker { progress.update(actor, new_state, upstream_total_key_count); if progress.is_done() { - tracing::debug!("all actors done for creating mview with epoch {}!", epoch); + tracing::debug!( + "all actors done for creating mview with table_id {}!", + table_id + ); // Clean-up the mapping from actors to DDL epoch. for actor in o.get().0.actors() { diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs index b3b57f2be58f1..21197a8df98d4 100644 --- a/src/meta/src/barrier/recovery.rs +++ b/src/meta/src/barrier/recovery.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant}; +use anyhow::anyhow; use futures::future::try_join_all; use itertools::Itertools; +use risingwave_common::catalog::TableId; use risingwave_pb::common::ActorInfo; use risingwave_pb::meta::PausedReason; use risingwave_pb::stream_plan::barrier::{BarrierKind, Mutation}; @@ -25,6 +27,7 @@ use risingwave_pb::stream_plan::AddMutation; use risingwave_pb::stream_service::{ BroadcastActorInfoTableRequest, BuildActorsRequest, ForceStopActorsRequest, UpdateActorsRequest, }; +use tokio::sync::oneshot; use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tracing::{debug, warn, Instrument}; use uuid::Uuid; @@ -32,6 +35,8 @@ use uuid::Uuid; use super::TracedEpoch; use crate::barrier::command::CommandContext; use crate::barrier::info::BarrierActorInfo; +use crate::barrier::notifier::Notifier; +use crate::barrier::progress::CreateMviewProgressTracker; use crate::barrier::{CheckpointControl, Command, GlobalBarrierManager}; use crate::manager::WorkerId; use crate::model::{BarrierManagerState, MigrationPlan}; @@ -60,22 +65,30 @@ impl GlobalBarrierManager { .await } + /// Please look at `CatalogManager::clean_dirty_tables` for more details. + /// This should only be called for bootstrap recovery. + async fn clean_dirty_tables(&self) -> MetaResult<()> { + let fragment_manager = self.fragment_manager.clone(); + self.catalog_manager + .clean_dirty_tables(fragment_manager) + .await?; + Ok(()) + } + /// Clean up all dirty streaming jobs. async fn clean_dirty_fragments(&self) -> MetaResult<()> { let stream_job_ids = self.catalog_manager.list_stream_job_ids().await?; let to_drop_table_fragments = self .fragment_manager - .list_dirty_table_fragments(|tf| { - !stream_job_ids.contains(&tf.table_id().table_id) || !tf.is_created() - }) + .list_dirty_table_fragments(|tf| !stream_job_ids.contains(&tf.table_id().table_id)) .await; - let to_drop_streaming_ids = to_drop_table_fragments .iter() .map(|t| t.table_id()) .collect(); debug!("clean dirty table fragments: {:?}", to_drop_streaming_ids); + self.fragment_manager .drop_table_fragments_vec(&to_drop_streaming_ids) .await?; @@ -86,7 +99,7 @@ impl GlobalBarrierManager { &to_drop_table_fragments ) .await.inspect_err(|e| - tracing::warn!( + warn!( "Failed to unregister compaction group for {:#?}. They will be cleaned up on node restart. {:#?}", to_drop_table_fragments, e) @@ -100,6 +113,101 @@ impl GlobalBarrierManager { Ok(()) } + async fn recover_background_mv_progress(&self) -> MetaResult<()> { + let creating_tables = self.catalog_manager.list_creating_background_mvs().await; + let creating_table_ids = creating_tables + .iter() + .map(|t| TableId { table_id: t.id }) + .collect_vec(); + + let mut senders = HashMap::new(); + let mut receivers = Vec::new(); + for table_id in creating_table_ids.iter().copied() { + let (finished_tx, finished_rx) = oneshot::channel(); + senders.insert( + table_id, + Notifier { + finished: Some(finished_tx), + ..Default::default() + }, + ); + + let fragments = self + .fragment_manager + .select_table_fragments_by_table_id(&table_id) + .await?; + let internal_table_ids = fragments.internal_table_ids(); + let internal_tables = self.catalog_manager.get_tables(&internal_table_ids).await; + let table = self.catalog_manager.get_tables(&[table_id.table_id]).await; + assert_eq!(table.len(), 1, "should only have 1 materialized table"); + let table = table.into_iter().next().unwrap(); + receivers.push((table, internal_tables, finished_rx)); + } + + let table_map = self + .fragment_manager + .get_table_id_actor_mapping(&creating_table_ids) + .await; + let table_fragment_map = self + .fragment_manager + .get_table_id_table_fragment_map(&creating_table_ids) + .await?; + let upstream_mv_counts = self + .fragment_manager + .get_upstream_relation_counts(&creating_table_ids) + .await; + let definitions: HashMap<_, _> = creating_tables + .into_iter() + .map(|t| (TableId { table_id: t.id }, t.definition)) + .collect(); + let version_stats = self.hummock_manager.get_version_stats().await; + // If failed, enter recovery mode. + { + let mut tracker = self.tracker.lock().await; + *tracker = CreateMviewProgressTracker::recover( + table_map.into(), + upstream_mv_counts.into(), + definitions.into(), + version_stats, + senders.into(), + table_fragment_map.into(), + self.fragment_manager.clone(), + ); + } + for (table, internal_tables, finished) in receivers { + let catalog_manager = self.catalog_manager.clone(); + tokio::spawn(async move { + let res: MetaResult<()> = try { + tracing::debug!("recovering stream job {}", table.id); + finished + .await + .map_err(|e| anyhow!("failed to finish command: {}", e))?; + + tracing::debug!("finished stream job {}", table.id); + // Once notified that job is finished we need to notify frontend. + // and mark catalog as created and commit to meta. + // both of these are done by catalog manager. + catalog_manager + .finish_create_table_procedure(internal_tables, table.clone()) + .await?; + tracing::debug!("notified frontend for stream job {}", table.id); + }; + if let Err(e) = res.as_ref() { + tracing::error!( + "stream job {} interrupted, will retry after recovery: {e:?}", + table.id + ); + // NOTE(kwannoel): We should not cleanup stream jobs, + // we don't know if it's just due to CN killed, + // or the job has actually failed. + // Users have to manually cancel the stream jobs, + // if they want to clean it. + } + }); + } + Ok(()) + } + /// Recovery the whole cluster from the latest epoch. /// /// If `paused_reason` is `Some`, all data sources (including connectors and DMLs) will be @@ -111,6 +219,7 @@ impl GlobalBarrierManager { &self, prev_epoch: TracedEpoch, paused_reason: Option, + bootstrap_recovery: bool, ) -> BarrierManagerState { // Mark blocked and abort buffered schedules, they might be dirty already. self.scheduled_barriers @@ -118,12 +227,25 @@ impl GlobalBarrierManager { .await; tracing::info!("recovery start!"); + if bootstrap_recovery { + self.clean_dirty_tables() + .await + .expect("clean dirty tables should not fail"); + } self.clean_dirty_fragments() .await .expect("clean dirty fragments"); + self.sink_manager.reset().await; let retry_strategy = Self::get_retry_strategy(); + // Mview progress needs to be recovered. + tracing::info!("recovering mview progress"); + self.recover_background_mv_progress() + .await + .expect("recover mview progress should not fail"); + tracing::info!("recovered mview progress"); + // We take retry into consideration because this is the latency user sees for a cluster to // get recovered. let recovery_timer = self.metrics.recovery_latency.start_timer(); @@ -172,6 +294,8 @@ impl GlobalBarrierManager { // Inject the `Initial` barrier to initialize all executors. let command_ctx = Arc::new(CommandContext::new( self.fragment_manager.clone(), + self.catalog_manager.clone(), + self.hummock_manager.clone(), self.env.stream_client_pool_ref(), info, prev_epoch.clone(), @@ -204,7 +328,7 @@ impl GlobalBarrierManager { warn!(err = ?err, "post_collect failed"); Err(err) } else { - Ok((new_epoch, response)) + Ok((new_epoch.clone(), response)) } } Err(err) => { diff --git a/src/meta/src/error.rs b/src/meta/src/error.rs index b056fea0e35e7..f8332819a4610 100644 --- a/src/meta/src/error.rs +++ b/src/meta/src/error.rs @@ -56,6 +56,9 @@ enum MetaErrorInner { #[error("{0} id not found: {1}")] CatalogIdNotFound(&'static str, u32), + #[error("table_fragment not exist: id={0}")] + FragmentNotFound(u32), + #[error("{0} with name {1} exists")] Duplicated(&'static str, String), @@ -135,6 +138,14 @@ impl MetaError { MetaErrorInner::CatalogIdNotFound(relation, id.into()).into() } + pub fn fragment_not_found>(id: T) -> Self { + MetaErrorInner::FragmentNotFound(id.into()).into() + } + + pub fn is_fragment_not_found(&self) -> bool { + matches!(self.inner.as_ref(), &MetaErrorInner::FragmentNotFound(..)) + } + pub fn catalog_duplicated>(relation: &'static str, name: T) -> Self { MetaErrorInner::Duplicated(relation, name.into()).into() } diff --git a/src/meta/src/manager/catalog/database.rs b/src/meta/src/manager/catalog/database.rs index ad1928e0bdd50..62b5692ce82ba 100644 --- a/src/meta/src/manager/catalog/database.rs +++ b/src/meta/src/manager/catalog/database.rs @@ -16,10 +16,12 @@ use std::collections::hash_map::Entry; use std::collections::{BTreeMap, HashMap, HashSet}; use itertools::Itertools; +use risingwave_common::bail; use risingwave_common::catalog::TableOption; use risingwave_pb::catalog::table::TableType; use risingwave_pb::catalog::{ - Connection, Database, Function, Index, PbStreamJobStatus, Schema, Sink, Source, Table, View, + Connection, CreateType, Database, Function, Index, PbStreamJobStatus, Schema, Sink, Source, + StreamJobStatus, Table, View, }; use super::{ConnectionId, DatabaseId, FunctionId, RelationId, SchemaId, SinkId, SourceId, ViewId}; @@ -194,12 +196,16 @@ impl DatabaseManager { } pub fn check_relation_name_duplicated(&self, relation_key: &RelationKey) -> MetaResult<()> { - if self.tables.values().any(|x| { + if let Some(t) = self.tables.values().find(|x| { x.database_id == relation_key.0 && x.schema_id == relation_key.1 && x.name.eq(&relation_key.2) }) { - Err(MetaError::catalog_duplicated("table", &relation_key.2)) + if t.stream_job_status == StreamJobStatus::Creating as i32 { + bail!("table is in creating procedure: {}", t.id); + } else { + Err(MetaError::catalog_duplicated("table", &relation_key.2)) + } } else if self.sources.values().any(|x| { x.database_id == relation_key.0 && x.schema_id == relation_key.1 @@ -258,9 +264,22 @@ impl DatabaseManager { self.databases.values().cloned().collect_vec() } - pub fn list_creating_tables(&self) -> Vec
{ - self.in_progress_creating_tables + pub fn list_creating_background_mvs(&self) -> Vec
{ + self.tables + .values() + .filter(|&t| { + t.stream_job_status == PbStreamJobStatus::Creating as i32 + && t.table_type == TableType::MaterializedView as i32 + && t.create_type == CreateType::Background as i32 + }) + .cloned() + .collect_vec() + } + + pub fn list_persisted_creating_tables(&self) -> Vec
{ + self.tables .values() + .filter(|&t| t.stream_job_status == PbStreamJobStatus::Creating as i32) .cloned() .collect_vec() } @@ -389,10 +408,12 @@ impl DatabaseManager { .contains(&relation.clone()) } + /// For all types of DDL pub fn mark_creating(&mut self, relation: &RelationKey) { self.in_progress_creation_tracker.insert(relation.clone()); } + /// Only for streaming DDL pub fn mark_creating_streaming_job(&mut self, table_id: TableId, key: RelationKey) { self.in_progress_creation_streaming_job .insert(table_id, key); @@ -417,6 +438,11 @@ impl DatabaseManager { self.in_progress_creation_streaming_job.keys().cloned() } + pub fn clear_creating_stream_jobs(&mut self) { + self.in_progress_creation_tracker.clear(); + self.in_progress_creation_streaming_job.clear(); + } + pub fn mark_creating_tables(&mut self, tables: &[Table]) { self.in_progress_creating_tables .extend(tables.iter().map(|t| (t.id, t.clone()))); diff --git a/src/meta/src/manager/catalog/fragment.rs b/src/meta/src/manager/catalog/fragment.rs index 1a74608c848a1..8b26b8afa11d9 100644 --- a/src/meta/src/manager/catalog/fragment.rs +++ b/src/meta/src/manager/catalog/fragment.rs @@ -43,7 +43,7 @@ use crate::model::{ }; use crate::storage::Transaction; use crate::stream::{SplitAssignment, TableRevision}; -use crate::MetaResult; +use crate::{MetaError, MetaResult}; pub struct FragmentManagerCore { table_fragments: BTreeMap, @@ -163,6 +163,56 @@ impl FragmentManager { map.values().cloned().collect() } + /// The `table_ids` here should correspond to stream jobs. + /// We get their corresponding table fragment, and from there, + /// we get the actors that are in the table fragment. + pub async fn get_table_id_actor_mapping( + &self, + table_ids: &[TableId], + ) -> HashMap> { + let map = &self.core.read().await.table_fragments; + let mut table_map = HashMap::new(); + for table_id in table_ids { + if let Some(table_fragment) = map.get(table_id) { + let mut actors = vec![]; + for fragment in table_fragment.fragments.values() { + for actor in &fragment.actors { + actors.push(actor.actor_id) + } + } + table_map.insert(*table_id, actors); + } + } + table_map + } + + /// Gets the counts for each upstream relation that each stream job + /// indicated by `table_ids` depends on. + /// For example in the following query: + /// ```sql + /// CREATE MATERIALIZED VIEW m1 AS + /// SELECT * FROM t1 JOIN t2 ON t1.a = t2.a JOIN t3 ON t2.b = t3.b + /// ``` + /// + /// We have t1 occurring once, and t2 occurring once. + pub async fn get_upstream_relation_counts( + &self, + table_ids: &[TableId], + ) -> HashMap> { + let map = &self.core.read().await.table_fragments; + let mut upstream_relation_counts = HashMap::new(); + for table_id in table_ids { + if let Some(table_fragments) = map.get(table_id) { + let dependent_ids = table_fragments.dependent_table_ids(); + let r = upstream_relation_counts.insert(*table_id, dependent_ids); + assert!(r.is_none(), "Each table_id should be unique!") + } else { + upstream_relation_counts.insert(*table_id, HashMap::new()); + } + } + upstream_relation_counts + } + pub fn get_mv_id_to_internal_table_ids_mapping(&self) -> Option)>> { match self.core.try_read() { Ok(core) => Some( @@ -231,10 +281,11 @@ impl FragmentManager { table_id: &TableId, ) -> MetaResult { let map = &self.core.read().await.table_fragments; - Ok(map - .get(table_id) - .cloned() - .with_context(|| format!("table_fragment not exist: id={}", table_id))?) + if let Some(table_fragment) = map.get(table_id) { + Ok(table_fragment.clone()) + } else { + Err(MetaError::fragment_not_found(table_id.table_id)) + } } pub async fn select_table_fragments_by_ids( @@ -244,15 +295,32 @@ impl FragmentManager { let map = &self.core.read().await.table_fragments; let mut table_fragments = Vec::with_capacity(table_ids.len()); for table_id in table_ids { - table_fragments.push( - map.get(table_id) - .cloned() - .with_context(|| format!("table_fragment not exist: id={}", table_id))?, - ); + table_fragments.push(if let Some(table_fragment) = map.get(table_id) { + table_fragment.clone() + } else { + return Err(MetaError::fragment_not_found(table_id.table_id)); + }); } Ok(table_fragments) } + pub async fn get_table_id_table_fragment_map( + &self, + table_ids: &[TableId], + ) -> MetaResult> { + let map = &self.core.read().await.table_fragments; + let mut id_to_fragment = HashMap::new(); + for table_id in table_ids { + let table_fragment = if let Some(table_fragment) = map.get(table_id) { + table_fragment.clone() + } else { + return Err(MetaError::fragment_not_found(table_id.table_id)); + }; + id_to_fragment.insert(*table_id, table_fragment); + } + Ok(id_to_fragment) + } + /// Start create a new `TableFragments` and insert it into meta store, currently the actors' /// state is `ActorState::Inactive` and the table fragments' state is `State::Initial`. pub async fn start_create_table_fragments( @@ -499,6 +567,8 @@ impl FragmentManager { /// Drop table fragments info and remove downstream actor infos in fragments from its dependent /// tables. + /// If table fragments already deleted, this should just be noop, + /// the delete function (`table_fragments.remove`) will not return an error. pub async fn drop_table_fragments_vec(&self, table_ids: &HashSet) -> MetaResult<()> { let mut guard = self.core.write().await; let current_revision = guard.table_revision; @@ -514,7 +584,7 @@ impl FragmentManager { table_fragments.remove(table_fragment.table_id()); let chain_actor_ids = table_fragment.chain_actor_ids(); let dependent_table_ids = table_fragment.dependent_table_ids(); - for dependent_table_id in dependent_table_ids { + for (dependent_table_id, _) in dependent_table_ids { if table_ids.contains(&dependent_table_id) { continue; } diff --git a/src/meta/src/manager/catalog/mod.rs b/src/meta/src/manager/catalog/mod.rs index be3ed42b01b4d..bcac32922d180 100644 --- a/src/meta/src/manager/catalog/mod.rs +++ b/src/meta/src/manager/catalog/mod.rs @@ -17,7 +17,7 @@ mod fragment; mod user; mod utils; -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::iter; use std::option::Option::Some; use std::sync::Arc; @@ -32,9 +32,10 @@ use risingwave_common::catalog::{ DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_ID, SYSTEM_SCHEMAS, }; use risingwave_common::{bail, ensure}; -use risingwave_pb::catalog::table::OptionalAssociatedSourceId; +use risingwave_pb::catalog::table::{OptionalAssociatedSourceId, TableType}; use risingwave_pb::catalog::{ - Connection, Database, Function, Index, PbStreamJobStatus, Schema, Sink, Source, Table, View, + Connection, CreateType, Database, Function, Index, PbStreamJobStatus, Schema, Sink, Source, + StreamJobStatus, Table, View, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::user::grant_privilege::{ActionWithGrantOption, Object}; @@ -44,7 +45,7 @@ use tokio::sync::{Mutex, MutexGuard}; use user::*; use crate::manager::{IdCategory, MetaSrvEnv, NotificationVersion, StreamingJob}; -use crate::model::{BTreeMapTransaction, MetadataModel, ValTransaction}; +use crate::model::{BTreeMapTransaction, MetadataModel, TableFragments, ValTransaction}; use crate::storage::Transaction; use crate::{MetaError, MetaResult}; @@ -115,6 +116,7 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_common::util::epoch::Epoch; use risingwave_pb::meta::cancel_creating_jobs_request::CreatingJobInfo; use risingwave_pb::meta::relation::RelationInfo; +use risingwave_pb::meta::table_fragments::State; use risingwave_pb::meta::{Relation, RelationGroup}; pub(crate) use {commit_meta, commit_meta_with_trx}; @@ -629,9 +631,13 @@ impl CatalogManager { pub async fn start_create_stream_job_procedure( &self, stream_job: &StreamingJob, + internal_tables: Vec
, ) -> MetaResult<()> { match stream_job { - StreamingJob::MaterializedView(table) => self.start_create_table_procedure(table).await, + StreamingJob::MaterializedView(table) => { + self.start_create_table_procedure(table, internal_tables) + .await + } StreamingJob::Sink(sink) => self.start_create_sink_procedure(sink).await, StreamingJob::Index(index, index_table) => { self.start_create_index_procedure(index, index_table).await @@ -641,7 +647,7 @@ impl CatalogManager { self.start_create_table_procedure_with_source(source, table) .await } else { - self.start_create_table_procedure(table).await + self.start_create_table_procedure(table, vec![]).await } } } @@ -694,7 +700,11 @@ impl CatalogManager { } /// This is used for both `CREATE TABLE` and `CREATE MATERIALIZED VIEW`. - pub async fn start_create_table_procedure(&self, table: &Table) -> MetaResult<()> { + pub async fn start_create_table_procedure( + &self, + table: &Table, + internal_tables: Vec
, + ) -> MetaResult<()> { let core = &mut *self.core.lock().await; let database_core = &mut core.database; let user_core = &mut core.user; @@ -707,19 +717,151 @@ impl CatalogManager { #[cfg(not(test))] user_core.ensure_user_id(table.owner)?; let key = (table.database_id, table.schema_id, table.name.clone()); + database_core.check_relation_name_duplicated(&key)?; - if database_core.has_in_progress_creation(&key) { - bail!("table is in creating procedure"); - } else { - database_core.mark_creating(&key); - database_core.mark_creating_streaming_job(table.id, key); - for &dependent_relation_id in &table.dependent_relations { - database_core.increase_ref_count(dependent_relation_id); + let mut tables = BTreeMapTransaction::new(&mut database_core.tables); + assert!( + !tables.contains_key(&table.id), + "table must not already exist in meta" + ); + for table in internal_tables { + tables.insert(table.id, table); + } + tables.insert(table.id, table.clone()); + commit_meta!(self, tables)?; + + for &dependent_relation_id in &table.dependent_relations { + database_core.increase_ref_count(dependent_relation_id); + } + user_core.increase_ref(table.owner); + Ok(()) + } + + fn assert_table_creating(tables: &BTreeMap, table: &Table) { + if let Some(t) = tables.get(&table.id) + && let Ok(StreamJobStatus::Creating) = t.get_stream_job_status() + {} else { + panic!("Table must be in creating procedure: {table:#?}") + } + } + + pub async fn assert_tables_deleted(&self, table_ids: Vec) { + let core = self.core.lock().await; + let tables = &core.database.tables; + for id in table_ids { + assert_eq!(tables.get(&id), None,) + } + } + + /// We clean the following tables: + /// 1. Those which belonged to incomplete Foreground jobs. + /// 2. Those which did not persist their table fragments, we can't recover these. + /// 3. Those which were only initialized, but not actually running yet. + /// 4. From 2, since we don't have internal table ids from the fragments, + /// we can detect hanging table ids by just finding all internal ids + /// with: + /// 1. `stream_job_status` = CREATING + /// 2. Not belonging to a background stream job. + /// Clean up these hanging tables by the id. + pub async fn clean_dirty_tables(&self, fragment_manager: FragmentManagerRef) -> MetaResult<()> { + let creating_tables: Vec
= self.list_persisted_creating_tables().await; + tracing::debug!( + "creating_tables ids: {:#?}", + creating_tables.iter().map(|t| t.id).collect_vec() + ); + let mut reserved_internal_tables = HashSet::new(); + let mut tables_to_clean = vec![]; + let mut internal_tables_to_clean = vec![]; + for table in creating_tables { + tracing::trace!( + "checking table {} definition: {}, create_type: {:#?}, table_type: {:#?}", + table.id, + table.definition, + table.get_create_type().unwrap_or(CreateType::Foreground), + table.get_table_type().unwrap(), + ); + // 1. Incomplete Foreground jobs + if table.create_type == CreateType::Foreground as i32 + && table.table_type != TableType::Internal as i32 + // || table.create_type == CreateType::Unspecified as i32 + { + tracing::debug!("cleaning table_id for foreground: {:#?}", table.id); + tables_to_clean.push(table); + continue; } - user_core.increase_ref(table.owner); - Ok(()) + if table.table_type == TableType::Internal as i32 { + internal_tables_to_clean.push(table); + continue; + } + + // 2. No table fragments + assert_ne!(table.table_type, TableType::Internal as i32); + match fragment_manager + .select_table_fragments_by_table_id(&table.id.into()) + .await + { + Err(e) => { + if e.is_fragment_not_found() { + tracing::debug!("cleaning table_id for no fragments: {:#?}", table.id); + tables_to_clean.push(table); + continue; + } else { + return Err(e); + } + } + Ok(fragment) => { + let fragment: TableFragments = fragment; + // 3. For those in initial state (i.e. not running / created), + // we should purge them. + if fragment.state() == State::Initial { + tracing::debug!("cleaning table_id no initial state: {:#?}", table.id); + tables_to_clean.push(table); + continue; + } else { + assert_eq!(table.create_type, CreateType::Background as i32); + // 4. Get all the corresponding internal tables, the rest we can purge. + for id in fragment.internal_table_ids() { + reserved_internal_tables.insert(id); + } + continue; + } + } + } + } + for t in internal_tables_to_clean { + if !reserved_internal_tables.contains(&t.id) { + tracing::debug!( + "cleaning table_id for internal tables not reserved: {:#?}", + t.id + ); + tables_to_clean.push(t); + } + } + + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + let tables = &mut database_core.tables; + let mut tables = BTreeMapTransaction::new(tables); + for table in &tables_to_clean { + tracing::debug!("cleaning table_id: {}", table.id); + let table = tables.remove(table.id); + assert!(table.is_some()) + } + commit_meta!(self, tables)?; + + database_core.clear_creating_stream_jobs(); + let user_core = &mut core.user; + for table in &tables_to_clean { + // Recovered when init database manager. + for relation_id in &table.dependent_relations { + database_core.decrease_ref_count(*relation_id); + } + // Recovered when init user manager. + user_core.decrease_ref(table.owner); } + + Ok(()) } /// This is used for both `CREATE TABLE` and `CREATE MATERIALIZED VIEW`. @@ -730,17 +872,11 @@ impl CatalogManager { ) -> MetaResult { let core = &mut *self.core.lock().await; let database_core = &mut core.database; - let mut tables = BTreeMapTransaction::new(&mut database_core.tables); - let key = (table.database_id, table.schema_id, table.name.clone()); - assert!( - !tables.contains_key(&table.id) - && database_core.in_progress_creation_tracker.contains(&key), - "table must be in creating procedure" - ); - database_core.in_progress_creation_tracker.remove(&key); - database_core - .in_progress_creation_streaming_job - .remove(&table.id); + let tables = &mut database_core.tables; + if cfg!(not(test)) { + Self::assert_table_creating(tables, &table); + } + let mut tables = BTreeMapTransaction::new(tables); table.stream_job_status = PbStreamJobStatus::Created.into(); tables.insert(table.id, table.clone()); @@ -769,23 +905,60 @@ impl CatalogManager { Ok(version) } - pub async fn cancel_create_table_procedure(&self, table: &Table) { - let core = &mut *self.core.lock().await; - let database_core = &mut core.database; - let user_core = &mut core.user; - let key = (table.database_id, table.schema_id, table.name.clone()); - assert!( - !database_core.tables.contains_key(&table.id) - && database_core.has_in_progress_creation(&key), - "table must be in creating procedure" - ); + /// Used to cleanup states in stream manager. + /// It is required because failure may not necessarily happen in barrier, + /// e.g. when cordon nodes. + /// and we still need some way to cleanup the state. + pub async fn cancel_create_table_procedure( + &self, + table_id: TableId, + internal_table_ids: Vec, + ) -> MetaResult<()> { + let table = { + let core = &mut self.core.lock().await; + let database_core = &mut core.database; + let tables = &mut database_core.tables; + let Some(table) = tables.get(&table_id).cloned() else { + bail!( + "table_id {} missing when attempting to cancel job", + table_id + ) + }; + table + }; - database_core.unmark_creating(&key); - database_core.unmark_creating_streaming_job(table.id); - for &dependent_relation_id in &table.dependent_relations { - database_core.decrease_ref_count(dependent_relation_id); + tracing::trace!("cleanup tables for {}", table.id); + { + let core = &mut self.core.lock().await; + let database_core = &mut core.database; + + let mut table_ids = vec![table.id]; + table_ids.extend(internal_table_ids); + + let tables = &mut database_core.tables; + let mut tables = BTreeMapTransaction::new(tables); + for table_id in table_ids { + tables.remove(table_id); + } + commit_meta!(self, tables)?; + } + + { + let core = &mut self.core.lock().await; + { + let user_core = &mut core.user; + user_core.decrease_ref(table.owner); + } + + { + let database_core = &mut core.database; + for &dependent_relation_id in &table.dependent_relations { + database_core.decrease_ref_count(dependent_relation_id); + } + } } - user_core.decrease_ref(table.owner); + + Ok(()) } /// return id of streaming jobs in the database which need to be dropped by stream manager. @@ -975,7 +1148,7 @@ impl CatalogManager { match drop_mode { DropMode::Restrict => { return Err(MetaError::permission_denied(format!( - "Fail to delete table `{}` because {} other relation(s) depend on it", + "Fail to delete index table `{}` because {} other relation(s) depend on it", index_table.name, ref_count ))); } @@ -2205,6 +2378,24 @@ impl CatalogManager { self.core.lock().await.database.list_tables() } + /// Lists table catalogs for mviews, without their internal tables. + pub async fn list_creating_background_mvs(&self) -> Vec
{ + self.core + .lock() + .await + .database + .list_creating_background_mvs() + } + + /// Lists table catalogs for all tables with `stream_job_status=CREATING`. + pub async fn list_persisted_creating_tables(&self) -> Vec
{ + self.core + .lock() + .await + .database + .list_persisted_creating_tables() + } + pub async fn get_all_table_options(&self) -> HashMap { self.core.lock().await.database.get_all_table_options() } diff --git a/src/meta/src/manager/streaming_job.rs b/src/meta/src/manager/streaming_job.rs index 611de4120a787..e02388eba4f3d 100644 --- a/src/meta/src/manager/streaming_job.rs +++ b/src/meta/src/manager/streaming_job.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use risingwave_common::catalog::TableVersionId; use risingwave_common::util::epoch::Epoch; -use risingwave_pb::catalog::{Index, Sink, Source, Table}; +use risingwave_pb::catalog::{CreateType, Index, Sink, Source, Table}; use crate::model::FragmentId; @@ -197,4 +197,13 @@ impl StreamingJob { None } } + + pub fn create_type(&self) -> CreateType { + match self { + Self::MaterializedView(table) => { + table.get_create_type().unwrap_or(CreateType::Foreground) + } + _ => CreateType::Foreground, + } + } } diff --git a/src/meta/src/model/stream.rs b/src/meta/src/model/stream.rs index 10fe5abe8aeaa..726bd7fcd8e73 100644 --- a/src/meta/src/model/stream.rs +++ b/src/meta/src/model/stream.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::ops::AddAssign; use itertools::Itertools; use risingwave_common::catalog::TableId; @@ -353,9 +354,12 @@ impl TableFragments { } /// Resolve dependent table - fn resolve_dependent_table(stream_node: &StreamNode, table_ids: &mut HashSet) { + fn resolve_dependent_table(stream_node: &StreamNode, table_ids: &mut HashMap) { if let Some(NodeBody::Chain(chain)) = stream_node.node_body.as_ref() { - table_ids.insert(TableId::new(chain.table_id)); + table_ids + .entry(TableId::new(chain.table_id)) + .or_default() + .add_assign(1); } for child in &stream_node.input { @@ -363,9 +367,10 @@ impl TableFragments { } } - /// Returns dependent table ids. - pub fn dependent_table_ids(&self) -> HashSet { - let mut table_ids = HashSet::new(); + /// Returns a mapping of dependent table ids of the `TableFragments` + /// to their corresponding count. + pub fn dependent_table_ids(&self) -> HashMap { + let mut table_ids = HashMap::new(); self.fragments.values().for_each(|fragment| { let actor = &fragment.actors[0]; Self::resolve_dependent_table(actor.nodes.as_ref().unwrap(), &mut table_ids); diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs index 2202e51ca6698..04b9729c5a5b8 100644 --- a/src/meta/src/rpc/ddl_controller.rs +++ b/src/meta/src/rpc/ddl_controller.rs @@ -414,6 +414,11 @@ impl DdlController { fragment_graph: StreamFragmentGraphProto, create_type: CreateType, ) -> MetaResult { + tracing::debug!( + id = stream_job.id(), + definition = stream_job.definition(), + "starting stream job", + ); let _permit = self .creating_streaming_job_permits .semaphore @@ -423,6 +428,8 @@ impl DdlController { let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; let env = StreamEnvironment::from_protobuf(fragment_graph.get_env().unwrap()); + + tracing::debug!(id = stream_job.id(), "preparing stream job"); let fragment_graph = self .prepare_stream_job(&mut stream_job, fragment_graph) .await?; @@ -432,6 +439,7 @@ impl DdlController { let mut internal_tables = vec![]; let result = try { + tracing::debug!(id = stream_job.id(), "building stream job"); let (ctx, table_fragments) = self .build_stream_job(env, &stream_job, fragment_graph) .await?; @@ -455,7 +463,7 @@ impl DdlController { let (ctx, table_fragments) = match result { Ok(r) => r, Err(e) => { - self.cancel_stream_job(&stream_job, internal_tables).await; + self.cancel_stream_job(&stream_job, internal_tables).await?; return Err(e); } }; @@ -467,7 +475,7 @@ impl DdlController { } CreateType::Background => { let ctrl = self.clone(); - let definition = stream_job.definition(); + let stream_job_id = stream_job.id(); let fut = async move { let result = ctrl .create_streaming_job_inner( @@ -478,9 +486,11 @@ impl DdlController { ) .await; match result { - Err(e) => tracing::error!(definition, error = ?e, "stream_job_error"), + Err(e) => { + tracing::error!(id=stream_job_id, error = ?e, "finish stream job failed") + } Ok(_) => { - tracing::info!(definition, "stream_job_ok") + tracing::info!(id = stream_job_id, "finish stream job succeeded") } } }; @@ -490,6 +500,7 @@ impl DdlController { } } + // We persist table fragments at this step. async fn create_streaming_job_inner( &self, stream_job: StreamingJob, @@ -497,15 +508,29 @@ impl DdlController { ctx: CreateStreamingJobContext, internal_tables: Vec
, ) -> MetaResult { + let job_id = stream_job.id(); + tracing::debug!(id = job_id, "creating stream job"); let result = self .stream_manager .create_streaming_job(table_fragments, ctx) .await; if let Err(e) = result { - self.cancel_stream_job(&stream_job, internal_tables).await; + match stream_job.create_type() { + // NOTE: This assumes that we will trigger recovery, + // and recover stream job progress. + CreateType::Background => { + tracing::error!(id = stream_job.id(), error = ?e, "finish stream job failed") + } + _ => { + self.cancel_stream_job(&stream_job, internal_tables).await?; + } + } return Err(e); }; - self.finish_stream_job(stream_job, internal_tables).await + tracing::debug!(id = job_id, "finishing stream job"); + let version = self.finish_stream_job(stream_job, internal_tables).await?; + tracing::debug!(id = job_id, "finished stream job"); + Ok(version) } async fn drop_streaming_job( @@ -570,6 +595,8 @@ impl DdlController { StreamFragmentGraph::new(fragment_graph, self.env.id_gen_manager_ref(), stream_job) .await?; + let internal_tables = fragment_graph.internal_tables().into_values().collect_vec(); + // 2. Set the graph-related fields and freeze the `stream_job`. stream_job.set_table_fragment_id(fragment_graph.table_fragment_id()); stream_job.set_dml_fragment_id(fragment_graph.dml_fragment_id()); @@ -577,7 +604,7 @@ impl DdlController { // 3. Mark current relation as "creating" and add reference count to dependent relations. self.catalog_manager - .start_create_stream_job_procedure(stream_job) + .start_create_stream_job_procedure(stream_job, internal_tables) .await?; Ok(fragment_graph) @@ -686,6 +713,7 @@ impl DdlController { table_properties: stream_job.properties(), definition: stream_job.definition(), mv_table_id: stream_job.mv_table(), + create_type: stream_job.create_type(), }; // 4. Mark creating tables, including internal tables and the table of the stream job. @@ -702,17 +730,27 @@ impl DdlController { Ok((ctx, table_fragments)) } - /// `cancel_stream_job` cancels a stream job and clean some states. - async fn cancel_stream_job(&self, stream_job: &StreamingJob, internal_tables: Vec
) { + /// This is NOT used by `CANCEL JOBS`. + /// It is used internally by `DdlController` to cancel and cleanup stream job. + async fn cancel_stream_job( + &self, + stream_job: &StreamingJob, + internal_tables: Vec
, + ) -> MetaResult<()> { let mut creating_internal_table_ids = internal_tables.into_iter().map(|t| t.id).collect_vec(); // 1. cancel create procedure. match stream_job { StreamingJob::MaterializedView(table) => { - creating_internal_table_ids.push(table.id); - self.catalog_manager - .cancel_create_table_procedure(table) + // barrier manager will do the cleanup. + let result = self + .catalog_manager + .cancel_create_table_procedure(table.id, creating_internal_table_ids.clone()) .await; + creating_internal_table_ids.push(table.id); + if let Err(e) = result { + tracing::warn!("Failed to cancel create table procedure, perhaps barrier manager has already cleaned it. Reason: {e:#?}"); + } } StreamingJob::Sink(sink) => { self.catalog_manager @@ -720,16 +758,23 @@ impl DdlController { .await; } StreamingJob::Table(source, table) => { - creating_internal_table_ids.push(table.id); if let Some(source) = source { self.catalog_manager .cancel_create_table_procedure_with_source(source, table) .await; } else { - self.catalog_manager - .cancel_create_table_procedure(table) + let result = self + .catalog_manager + .cancel_create_table_procedure( + table.id, + creating_internal_table_ids.clone(), + ) .await; + if let Err(e) = result { + tracing::warn!("Failed to cancel create table procedure, perhaps barrier manager has already cleaned it. Reason: {e:#?}"); + } } + creating_internal_table_ids.push(table.id); } StreamingJob::Index(index, table) => { creating_internal_table_ids.push(table.id); @@ -742,6 +787,7 @@ impl DdlController { self.catalog_manager .unmark_creating_tables(&creating_internal_table_ids, true) .await; + Ok(()) } /// `finish_stream_job` finishes a stream job and clean some states. diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs index 44198f65635cb..77a784c64ac09 100644 --- a/src/meta/src/stream/stream_manager.rs +++ b/src/meta/src/stream/stream_manager.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use futures::future::{join_all, try_join_all, BoxFuture}; use itertools::Itertools; use risingwave_common::catalog::TableId; -use risingwave_pb::catalog::Table; +use risingwave_pb::catalog::{CreateType, Table}; use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use risingwave_pb::stream_plan::Dispatcher; use risingwave_pb::stream_service::{ @@ -67,6 +67,8 @@ pub struct CreateStreamingJobContext { pub definition: String, pub mv_table_id: Option, + + pub create_type: CreateType, } impl CreateStreamingJobContext { @@ -112,22 +114,32 @@ impl CreatingStreamingJobInfo { jobs.remove(&job_id); } - async fn cancel_jobs(&self, job_ids: Vec) -> HashMap> { + async fn cancel_jobs( + &self, + job_ids: Vec, + ) -> (HashMap>, Vec) { let mut jobs = self.streaming_jobs.lock().await; let mut receivers = HashMap::new(); + let mut recovered_job_ids = vec![]; for job_id in job_ids { if let Some(job) = jobs.get_mut(&job_id) && let Some(shutdown_tx) = job.shutdown_tx.take() { let (tx, rx) = oneshot::channel(); - if shutdown_tx.send(CreatingState::Canceling{finish_tx: tx}).await.is_ok() { + if shutdown_tx.send(CreatingState::Canceling { finish_tx: tx }).await.is_ok() { receivers.insert(job_id, rx); } else { tracing::warn!("failed to send canceling state"); } + } else { + // If these job ids do not exist in streaming_jobs, + // we can infer they either: + // 1. are entirely non-existent, + // 2. OR they are recovered streaming jobs, and managed by BarrierManager. + recovered_job_ids.push(job_id); } } - receivers + (receivers, recovered_job_ids) } } @@ -407,7 +419,7 @@ impl GlobalStreamManager { definition, mv_table_id, internal_tables, - .. + create_type, }: CreateStreamingJobContext, ) -> MetaResult<()> { // Register to compaction group beforehand. @@ -424,8 +436,10 @@ impl GlobalStreamManager { table_fragments.internal_table_ids().len() + mv_table_id.map_or(0, |_| 1) ); revert_funcs.push(Box::pin(async move { - if let Err(e) = hummock_manager_ref.unregister_table_ids(®istered_table_ids).await { - tracing::warn!("Failed to unregister compaction group for {:#?}. They will be cleaned up on node restart. {:#?}", registered_table_ids, e); + if create_type == CreateType::Foreground { + if let Err(e) = hummock_manager_ref.unregister_table_ids(®istered_table_ids).await { + tracing::warn!("Failed to unregister compaction group for {:#?}. They will be cleaned up on node restart. {:#?}", registered_table_ids, e); + } } })); @@ -452,9 +466,11 @@ impl GlobalStreamManager { }) .await { - self.fragment_manager - .drop_table_fragments_vec(&HashSet::from_iter(std::iter::once(table_id))) - .await?; + if create_type == CreateType::Foreground { + self.fragment_manager + .drop_table_fragments_vec(&HashSet::from_iter(std::iter::once(table_id))) + .await?; + } return Err(err); } @@ -553,13 +569,18 @@ impl GlobalStreamManager { } /// Cancel streaming jobs and return the canceled table ids. + /// 1. Send cancel message to stream jobs (via `cancel_jobs`). + /// 2. Send cancel message to recovered stream jobs (via `barrier_scheduler`). + /// + /// Cleanup of their state will be cleaned up after the `CancelStreamJob` command succeeds, + /// by the barrier manager for both of them. pub async fn cancel_streaming_jobs(&self, table_ids: Vec) -> Vec { if table_ids.is_empty() { return vec![]; } let _reschedule_job_lock = self.reschedule_lock.read().await; - let receivers = self.creating_job_info.cancel_jobs(table_ids).await; + let (receivers, recovered_job_ids) = self.creating_job_info.cancel_jobs(table_ids).await; let futures = receivers.into_iter().map(|(id, receiver)| async move { if receiver.await.is_ok() { @@ -570,7 +591,35 @@ impl GlobalStreamManager { None } }); - join_all(futures).await.into_iter().flatten().collect_vec() + let mut cancelled_ids = join_all(futures).await.into_iter().flatten().collect_vec(); + + // NOTE(kwannoel): For recovered stream jobs, we can directly cancel them by running the barrier command, + // since Barrier manager manages the recovered stream jobs. + let futures = recovered_job_ids.into_iter().map(|id| async move { + let result: MetaResult<()> = try { + let fragment = self + .fragment_manager + .select_table_fragments_by_table_id(&id) + .await?; + self.barrier_scheduler + .run_command(Command::CancelStreamingJob(fragment)) + .await?; + }; + match result { + Ok(_) => { + tracing::info!("cancelled recovered streaming job {id}"); + Some(id) + }, + Err(_) => { + tracing::error!("failed to cancel recovered streaming job {id}, does {id} correspond to any jobs in `SHOW JOBS`?"); + None + }, + } + }); + let cancelled_recovered_ids = join_all(futures).await.into_iter().flatten().collect_vec(); + + cancelled_ids.extend(cancelled_recovered_ids); + cancelled_ids } } @@ -896,7 +945,7 @@ mod tests { }; self.catalog_manager - .start_create_table_procedure(&table) + .start_create_table_procedure(&table, vec![]) .await?; self.global_stream_manager .create_streaming_job(table_fragments, ctx) diff --git a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs new file mode 100644 index 0000000000000..1fd5c90e59e4b --- /dev/null +++ b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs @@ -0,0 +1,96 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use anyhow::Result; +use risingwave_simulation::cluster::{Cluster, Configuration, KillOpts}; +use risingwave_simulation::utils::AssertResult; +use tokio::time::sleep; + +async fn kill_cn_and_wait_recover(cluster: &Cluster) { + // Kill it again + for _ in 0..5 { + cluster + .kill_node(&KillOpts { + kill_rate: 1.0, + kill_meta: false, + kill_frontend: false, + kill_compute: true, + kill_compactor: false, + restart_delay_secs: 1, + }) + .await; + sleep(Duration::from_secs(2)).await; + } + sleep(Duration::from_secs(10)).await; +} + +async fn kill_and_wait_recover(cluster: &Cluster) { + // Kill it again + for _ in 0..5 { + sleep(Duration::from_secs(2)).await; + cluster.kill_node(&KillOpts::ALL).await; + } + sleep(Duration::from_secs(20)).await; +} + +#[tokio::test] +async fn test_background_mv_barrier_recovery() -> Result<()> { + let mut cluster = Cluster::start(Configuration::for_backfill()).await?; + let mut session = cluster.start_session(); + + session.run("CREATE TABLE t1 (v1 int);").await?; + session + .run("INSERT INTO t1 select * from generate_series(1, 400000)") + .await?; + session.run("flush").await?; + session.run("SET BACKGROUND_DDL=true;").await?; + session + .run("create materialized view m1 as select * from t1;") + .await?; + + kill_cn_and_wait_recover(&cluster).await; + + // Send some upstream updates. + cluster + .run("INSERT INTO t1 select * from generate_series(1, 100000);") + .await?; + cluster.run("flush;").await?; + + kill_cn_and_wait_recover(&cluster).await; + + kill_and_wait_recover(&cluster).await; + + // Send some upstream updates. + cluster + .run("INSERT INTO t1 select * from generate_series(1, 100000);") + .await?; + cluster.run("flush;").await?; + + // Now just wait for it to complete. + + sleep(Duration::from_secs(10)).await; + + // Make sure after finished, we should have 5000_000 rows. + session + .run("SELECT COUNT(v1) FROM m1") + .await? + .assert_result_eq("600000"); + + session.run("DROP MATERIALIZED VIEW m1").await?; + session.run("DROP TABLE t1").await?; + + Ok(()) +} diff --git a/src/tests/simulation/tests/integration_tests/recovery/mod.rs b/src/tests/simulation/tests/integration_tests/recovery/mod.rs index 565487e8d7dbd..2430daad760a1 100644 --- a/src/tests/simulation/tests/integration_tests/recovery/mod.rs +++ b/src/tests/simulation/tests/integration_tests/recovery/mod.rs @@ -13,5 +13,6 @@ // limitations under the License. mod backfill; +mod background_ddl; mod nexmark_recovery; mod pause_on_bootstrap; From 19643adfc5e7ccb5706b99cafe5bd756072d5966 Mon Sep 17 00:00:00 2001 From: xiangjinwu <17769960+xiangjinwu@users.noreply.github.com> Date: Sat, 21 Oct 2023 16:15:40 +0800 Subject: [PATCH 16/16] refactor(source): prefer `SpecificParserConfig` over `SourceStruct` (#12602) --- src/batch/src/executor/source.rs | 5 +- src/connector/src/parser/avro/parser.rs | 11 +-- .../src/parser/debezium/avro_parser.rs | 11 +-- src/connector/src/parser/mod.rs | 50 +--------- src/connector/src/parser/protobuf/parser.rs | 36 +++----- src/connector/src/source/base.rs | 63 ++++++++++++- .../src/source/datagen/source/generator.rs | 28 +++--- .../src/source/datagen/source/reader.rs | 2 +- .../src/handler/alter_source_column.rs | 3 +- src/frontend/src/handler/create_source.rs | 37 ++------ src/source/src/source_desc.rs | 91 +++---------------- 11 files changed, 132 insertions(+), 205 deletions(-) diff --git a/src/batch/src/executor/source.rs b/src/batch/src/executor/source.rs index 71cac6e917dc3..8bf9fc5b7e610 100644 --- a/src/batch/src/executor/source.rs +++ b/src/batch/src/executor/source.rs @@ -30,7 +30,6 @@ use risingwave_connector::source::{ }; use risingwave_pb::batch_plan::plan_node::NodeBody; use risingwave_source::connector_source::ConnectorSource; -use risingwave_source::source_desc::extract_source_struct; use super::Executor; use crate::error::BatchError; @@ -71,9 +70,7 @@ impl BoxedExecutorBuilder for SourceExecutor { .map_err(|e| RwError::from(ConnectorError(e.into())))?; let info = source_node.get_info().unwrap(); - let source_struct = extract_source_struct(info)?; - let parser_config = - SpecificParserConfig::new(source_struct, info, &source_node.properties)?; + let parser_config = SpecificParserConfig::new(info, &source_node.properties)?; let columns: Vec<_> = source_node .columns diff --git a/src/connector/src/parser/avro/parser.rs b/src/connector/src/parser/avro/parser.rs index d583ef83a8227..3fcfc01cd9586 100644 --- a/src/connector/src/parser/avro/parser.rs +++ b/src/connector/src/parser/avro/parser.rs @@ -220,6 +220,7 @@ mod test { use risingwave_common::types::{DataType, Date, Interval, ScalarImpl, Timestamptz}; use risingwave_common::{error, try_match_expand}; use risingwave_pb::catalog::StreamSourceInfo; + use risingwave_pb::plan_common::{PbEncodeType, PbFormatType}; use url::Url; use super::{ @@ -232,7 +233,7 @@ mod test { use crate::parser::{ AccessBuilderImpl, EncodingType, SourceStreamChunkBuilder, SpecificParserConfig, }; - use crate::source::{SourceColumnDesc, SourceEncode, SourceFormat, SourceStruct}; + use crate::source::SourceColumnDesc; fn test_data_path(file_name: &str) -> String { let curr_dir = env::current_dir().unwrap().into_os_string(); @@ -304,13 +305,11 @@ mod test { let info = StreamSourceInfo { row_schema_location: schema_path.clone(), use_schema_registry: false, + format: PbFormatType::Plain.into(), + row_encode: PbEncodeType::Avro.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Avro), - &info, - &HashMap::new(), - )?; + let parser_config = SpecificParserConfig::new(&info, &HashMap::new())?; AvroParserConfig::new(parser_config.encoding_config).await } diff --git a/src/connector/src/parser/debezium/avro_parser.rs b/src/connector/src/parser/debezium/avro_parser.rs index 23161ba635b44..bd573ad3d0525 100644 --- a/src/connector/src/parser/debezium/avro_parser.rs +++ b/src/connector/src/parser/debezium/avro_parser.rs @@ -151,12 +151,13 @@ mod tests { use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_pb::catalog::StreamSourceInfo; + use risingwave_pb::plan_common::{PbEncodeType, PbFormatType}; use super::*; use crate::parser::{ DebeziumAvroParserConfig, DebeziumParser, SourceStreamChunkBuilder, SpecificParserConfig, }; - use crate::source::{SourceColumnDesc, SourceEncode, SourceFormat, SourceStruct}; + use crate::source::SourceColumnDesc; const DEBEZIUM_AVRO_DATA: &[u8] = b"\x00\x00\x00\x00\x06\x00\x02\xd2\x0f\x0a\x53\x61\x6c\x6c\x79\x0c\x54\x68\x6f\x6d\x61\x73\x2a\x73\x61\x6c\x6c\x79\x2e\x74\x68\x6f\x6d\x61\x73\x40\x61\x63\x6d\x65\x2e\x63\x6f\x6d\x16\x32\x2e\x31\x2e\x32\x2e\x46\x69\x6e\x61\x6c\x0a\x6d\x79\x73\x71\x6c\x12\x64\x62\x73\x65\x72\x76\x65\x72\x31\xc0\xb4\xe8\xb7\xc9\x61\x00\x30\x66\x69\x72\x73\x74\x5f\x69\x6e\x5f\x64\x61\x74\x61\x5f\x63\x6f\x6c\x6c\x65\x63\x74\x69\x6f\x6e\x12\x69\x6e\x76\x65\x6e\x74\x6f\x72\x79\x00\x02\x12\x63\x75\x73\x74\x6f\x6d\x65\x72\x73\x00\x00\x20\x6d\x79\x73\x71\x6c\x2d\x62\x69\x6e\x2e\x30\x30\x30\x30\x30\x33\x8c\x06\x00\x00\x00\x02\x72\x02\x92\xc3\xe8\xb7\xc9\x61\x00"; @@ -298,13 +299,11 @@ mod tests { )); let info = StreamSourceInfo { row_schema_location: "http://127.0.0.1:8081".into(), + format: PbFormatType::Debezium.into(), + row_encode: PbEncodeType::Avro.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Debezium, SourceEncode::Avro), - &info, - &props, - )?; + let parser_config = SpecificParserConfig::new(&info, &props)?; let config = DebeziumAvroParserConfig::new(parser_config.clone().encoding_config).await?; let columns = config .map_to_columns()? diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs index 2ff661447d1b1..c4f955492cc9a 100644 --- a/src/connector/src/parser/mod.rs +++ b/src/connector/src/parser/mod.rs @@ -47,8 +47,8 @@ use self::util::get_kafka_topic; use crate::aws_auth::AwsAuthProps; use crate::parser::maxwell::MaxwellParser; use crate::source::{ - BoxSourceStream, SourceColumnDesc, SourceColumnType, SourceContext, SourceContextRef, - SourceEncode, SourceFormat, SourceMeta, SourceStruct, SourceWithStateStream, SplitId, + extract_source_struct, BoxSourceStream, SourceColumnDesc, SourceColumnType, SourceContext, + SourceContextRef, SourceEncode, SourceFormat, SourceMeta, SourceWithStateStream, SplitId, StreamChunkWithState, }; @@ -870,35 +870,9 @@ pub enum ProtocolProperties { } impl SpecificParserConfig { - pub fn get_source_struct(&self) -> SourceStruct { - let format = match self.protocol_config { - ProtocolProperties::Debezium => SourceFormat::Debezium, - ProtocolProperties::DebeziumMongo => SourceFormat::DebeziumMongo, - ProtocolProperties::Maxwell => SourceFormat::Maxwell, - ProtocolProperties::Canal => SourceFormat::Canal, - ProtocolProperties::Plain => SourceFormat::Plain, - ProtocolProperties::Upsert => SourceFormat::Upsert, - ProtocolProperties::Native => SourceFormat::Native, - ProtocolProperties::Unspecified => unreachable!(), - }; - let encode = match self.encoding_config { - EncodingProperties::Avro(_) => SourceEncode::Avro, - EncodingProperties::Protobuf(_) => SourceEncode::Protobuf, - EncodingProperties::Csv(_) => SourceEncode::Csv, - EncodingProperties::Json(_) => SourceEncode::Json, - EncodingProperties::Bytes(_) => SourceEncode::Bytes, - EncodingProperties::Native => SourceEncode::Native, - EncodingProperties::Unspecified => unreachable!(), - }; - SourceStruct { format, encode } - } - // The validity of (format, encode) is ensured by `extract_format_encode` - pub fn new( - source_struct: SourceStruct, - info: &StreamSourceInfo, - props: &HashMap, - ) -> Result { + pub fn new(info: &StreamSourceInfo, props: &HashMap) -> Result { + let source_struct = extract_source_struct(info)?; let format = source_struct.format; let encode = source_struct.encode; // this transformation is needed since there may be config for the protocol @@ -1026,19 +1000,3 @@ impl SpecificParserConfig { }) } } - -impl ParserConfig { - pub fn new( - source_struct: SourceStruct, - info: &StreamSourceInfo, - props: &HashMap, - rw_columns: &Vec, - ) -> Result { - let common = CommonParserConfig { - rw_columns: rw_columns.to_owned(), - }; - let specific = SpecificParserConfig::new(source_struct, info, props)?; - - Ok(Self { common, specific }) - } -} diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs index cc8dc0734bdfe..b2ee1c23bca04 100644 --- a/src/connector/src/parser/protobuf/parser.rs +++ b/src/connector/src/parser/protobuf/parser.rs @@ -358,13 +358,13 @@ mod test { use risingwave_common::types::{DataType, StructType}; use risingwave_pb::catalog::StreamSourceInfo; use risingwave_pb::data::data_type::PbTypeName; + use risingwave_pb::plan_common::{PbEncodeType, PbFormatType}; use super::*; use crate::parser::protobuf::recursive::all_types::{EnumType, ExampleOneof, NestedMessage}; use crate::parser::protobuf::recursive::AllTypes; use crate::parser::unified::Access; use crate::parser::SpecificParserConfig; - use crate::source::{SourceEncode, SourceFormat, SourceStruct}; fn schema_dir() -> String { let dir = PathBuf::from("src/test_data"); @@ -391,13 +391,11 @@ mod test { proto_message_name: message_name.to_string(), row_schema_location: location.to_string(), use_schema_registry: false, + format: PbFormatType::Plain.into(), + row_encode: PbEncodeType::Protobuf.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Protobuf), - &info, - &HashMap::new(), - )?; + let parser_config = SpecificParserConfig::new(&info, &HashMap::new())?; let conf = ProtobufParserConfig::new(parser_config.encoding_config).await?; let value = DynamicMessage::decode(conf.message_descriptor, PRE_GEN_PROTO_DATA).unwrap(); @@ -438,13 +436,11 @@ mod test { proto_message_name: message_name.to_string(), row_schema_location: location.to_string(), use_schema_registry: false, + format: PbFormatType::Plain.into(), + row_encode: PbEncodeType::Protobuf.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Protobuf), - &info, - &HashMap::new(), - )?; + let parser_config = SpecificParserConfig::new(&info, &HashMap::new())?; let conf = ProtobufParserConfig::new(parser_config.encoding_config).await?; let columns = conf.map_to_columns().unwrap(); @@ -489,14 +485,11 @@ mod test { proto_message_name: message_name.to_string(), row_schema_location: location.to_string(), use_schema_registry: false, + format: PbFormatType::Plain.into(), + row_encode: PbEncodeType::Protobuf.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Protobuf), - &info, - &HashMap::new(), - ) - .unwrap(); + let parser_config = SpecificParserConfig::new(&info, &HashMap::new()).unwrap(); let conf = ProtobufParserConfig::new(parser_config.encoding_config) .await .unwrap(); @@ -518,14 +511,11 @@ mod test { proto_message_name: message_name.to_string(), row_schema_location: location.to_string(), use_schema_registry: false, + format: PbFormatType::Plain.into(), + row_encode: PbEncodeType::Protobuf.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Protobuf), - &info, - &HashMap::new(), - ) - .unwrap(); + let parser_config = SpecificParserConfig::new(&info, &HashMap::new()).unwrap(); ProtobufParserConfig::new(parser_config.encoding_config) .await diff --git a/src/connector/src/source/base.rs b/src/connector/src/source/base.rs index a0c0aee86592e..6a8cd12ce9fac 100644 --- a/src/connector/src/source/base.rs +++ b/src/connector/src/source/base.rs @@ -28,7 +28,7 @@ use risingwave_common::array::StreamChunk; use risingwave_common::catalog::TableId; use risingwave_common::error::{ErrorSuppressor, RwError}; use risingwave_common::types::{JsonbVal, Scalar}; -use risingwave_pb::catalog::PbSource; +use risingwave_pb::catalog::{PbSource, PbStreamSourceInfo}; use risingwave_pb::source::ConnectorSplit; use risingwave_rpc_client::ConnectorClient; use serde::de::DeserializeOwned; @@ -253,6 +253,67 @@ impl SourceStruct { } } +// Only return valid (format, encode) +pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result { + use risingwave_pb::plan_common::{PbEncodeType, PbFormatType, RowFormatType}; + + // old version meta. + if let Ok(format) = info.get_row_format() { + let (format, encode) = match format { + RowFormatType::Json => (SourceFormat::Plain, SourceEncode::Json), + RowFormatType::Protobuf => (SourceFormat::Plain, SourceEncode::Protobuf), + RowFormatType::DebeziumJson => (SourceFormat::Debezium, SourceEncode::Json), + RowFormatType::Avro => (SourceFormat::Plain, SourceEncode::Avro), + RowFormatType::Maxwell => (SourceFormat::Maxwell, SourceEncode::Json), + RowFormatType::CanalJson => (SourceFormat::Canal, SourceEncode::Json), + RowFormatType::Csv => (SourceFormat::Plain, SourceEncode::Csv), + RowFormatType::Native => (SourceFormat::Native, SourceEncode::Native), + RowFormatType::DebeziumAvro => (SourceFormat::Debezium, SourceEncode::Avro), + RowFormatType::UpsertJson => (SourceFormat::Upsert, SourceEncode::Json), + RowFormatType::UpsertAvro => (SourceFormat::Upsert, SourceEncode::Avro), + RowFormatType::DebeziumMongoJson => (SourceFormat::DebeziumMongo, SourceEncode::Json), + RowFormatType::Bytes => (SourceFormat::Plain, SourceEncode::Bytes), + RowFormatType::RowUnspecified => unreachable!(), + }; + return Ok(SourceStruct::new(format, encode)); + } + let source_format = info.get_format().map_err(|e| anyhow!("{e:?}"))?; + let source_encode = info.get_row_encode().map_err(|e| anyhow!("{e:?}"))?; + let (format, encode) = match (source_format, source_encode) { + (PbFormatType::Plain, PbEncodeType::Json) => (SourceFormat::Plain, SourceEncode::Json), + (PbFormatType::Plain, PbEncodeType::Protobuf) => { + (SourceFormat::Plain, SourceEncode::Protobuf) + } + (PbFormatType::Debezium, PbEncodeType::Json) => { + (SourceFormat::Debezium, SourceEncode::Json) + } + (PbFormatType::Plain, PbEncodeType::Avro) => (SourceFormat::Plain, SourceEncode::Avro), + (PbFormatType::Maxwell, PbEncodeType::Json) => (SourceFormat::Maxwell, SourceEncode::Json), + (PbFormatType::Canal, PbEncodeType::Json) => (SourceFormat::Canal, SourceEncode::Json), + (PbFormatType::Plain, PbEncodeType::Csv) => (SourceFormat::Plain, SourceEncode::Csv), + (PbFormatType::Native, PbEncodeType::Native) => { + (SourceFormat::Native, SourceEncode::Native) + } + (PbFormatType::Debezium, PbEncodeType::Avro) => { + (SourceFormat::Debezium, SourceEncode::Avro) + } + (PbFormatType::Upsert, PbEncodeType::Json) => (SourceFormat::Upsert, SourceEncode::Json), + (PbFormatType::Upsert, PbEncodeType::Avro) => (SourceFormat::Upsert, SourceEncode::Avro), + (PbFormatType::DebeziumMongo, PbEncodeType::Json) => { + (SourceFormat::DebeziumMongo, SourceEncode::Json) + } + (PbFormatType::Plain, PbEncodeType::Bytes) => (SourceFormat::Plain, SourceEncode::Bytes), + (format, encode) => { + return Err(anyhow!( + "Unsupported combination of format {:?} and encode {:?}", + format, + encode + )); + } + }; + Ok(SourceStruct::new(format, encode)) +} + pub type BoxSourceStream = BoxStream<'static, Result>>; pub trait SourceWithStateStream = diff --git a/src/connector/src/source/datagen/source/generator.rs b/src/connector/src/source/datagen/source/generator.rs index c073403a7565d..3e42b07ce4422 100644 --- a/src/connector/src/source/datagen/source/generator.rs +++ b/src/connector/src/source/datagen/source/generator.rs @@ -23,10 +23,8 @@ use risingwave_common::row::OwnedRow; use risingwave_common::types::DataType; use risingwave_common::util::iter_util::ZipEqFast; -use crate::source::{ - SourceEncode, SourceFormat, SourceMessage, SourceMeta, SourceStruct, SplitId, - StreamChunkWithState, -}; +use crate::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig}; +use crate::source::{SourceMessage, SourceMeta, SplitId, StreamChunkWithState}; pub enum FieldDesc { // field is invisible, generate None @@ -38,7 +36,7 @@ pub struct DatagenEventGenerator { // fields_map: HashMap, field_names: Vec, fields_vec: Vec, - source_struct: SourceStruct, + source_format: SpecificParserConfig, data_types: Vec, offset: u64, split_id: SplitId, @@ -56,7 +54,7 @@ impl DatagenEventGenerator { pub fn new( fields_vec: Vec, field_names: Vec, - source_struct: SourceStruct, + source_format: SpecificParserConfig, data_types: Vec, rows_per_second: u64, offset: u64, @@ -72,7 +70,7 @@ impl DatagenEventGenerator { Ok(Self { field_names, fields_vec, - source_struct, + source_format, data_types, offset, split_id, @@ -96,8 +94,11 @@ impl DatagenEventGenerator { ); let mut msgs = Vec::with_capacity(num_rows_to_generate as usize); 'outer: for _ in 0..num_rows_to_generate { - let payload = match (self.source_struct.format, self.source_struct.encode) { - (SourceFormat::Plain, SourceEncode::Json) => { + let payload = match ( + &self.source_format.protocol_config, + &self.source_format.encoding_config, + ) { + (ProtocolProperties::Plain, EncodingProperties::Json(_)) => { let mut map = serde_json::Map::with_capacity(self.fields_vec.len()); for (name, field_generator) in self .field_names @@ -225,7 +226,6 @@ mod tests { use futures::stream::StreamExt; use super::*; - use crate::source::SourceEncode; async fn check_sequence_partition_result( split_num: u64, @@ -266,7 +266,13 @@ mod tests { let generator = DatagenEventGenerator::new( fields_vec, vec!["c1".to_owned(), "c2".to_owned()], - SourceStruct::new(SourceFormat::Plain, SourceEncode::Json), + SpecificParserConfig { + protocol_config: ProtocolProperties::Plain, + encoding_config: EncodingProperties::Json(crate::parser::JsonProperties { + use_schema_registry: false, + }), + key_encoding_config: None, + }, data_types, rows_per_second, 0, diff --git a/src/connector/src/source/datagen/source/reader.rs b/src/connector/src/source/datagen/source/reader.rs index 3840b0e1f5dbc..bd9f74ee3aa9a 100644 --- a/src/connector/src/source/datagen/source/reader.rs +++ b/src/connector/src/source/datagen/source/reader.rs @@ -120,7 +120,7 @@ impl SplitReader for DatagenSplitReader { let generator = DatagenEventGenerator::new( fields_vec, field_names, - parser_config.specific.get_source_struct(), + parser_config.specific.clone(), data_types, rows_per_second, events_so_far, diff --git a/src/frontend/src/handler/alter_source_column.rs b/src/frontend/src/handler/alter_source_column.rs index 6e13a16185bf2..385a1010b50c9 100644 --- a/src/frontend/src/handler/alter_source_column.rs +++ b/src/frontend/src/handler/alter_source_column.rs @@ -16,8 +16,7 @@ use itertools::Itertools; use pgwire::pg_response::{PgResponse, StatementType}; use risingwave_common::catalog::ColumnId; use risingwave_common::error::{ErrorCode, Result, RwError}; -use risingwave_connector::source::{SourceEncode, SourceStruct}; -use risingwave_source::source_desc::extract_source_struct; +use risingwave_connector::source::{extract_source_struct, SourceEncode, SourceStruct}; use risingwave_sqlparser::ast::{ AlterSourceOperation, ColumnDef, CreateSourceStatement, ObjectName, Statement, }; diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs index d585caf70a920..bb02553e2cd9c 100644 --- a/src/frontend/src/handler/create_source.rs +++ b/src/frontend/src/handler/create_source.rs @@ -37,9 +37,8 @@ use risingwave_connector::source::datagen::DATAGEN_CONNECTOR; use risingwave_connector::source::nexmark::source::{get_event_data_types_with_names, EventType}; use risingwave_connector::source::test_source::TEST_CONNECTOR; use risingwave_connector::source::{ - SourceEncode, SourceFormat, SourceStruct, GOOGLE_PUBSUB_CONNECTOR, KAFKA_CONNECTOR, - KINESIS_CONNECTOR, NATS_CONNECTOR, NEXMARK_CONNECTOR, PULSAR_CONNECTOR, S3_CONNECTOR, - S3_V2_CONNECTOR, + GOOGLE_PUBSUB_CONNECTOR, KAFKA_CONNECTOR, KINESIS_CONNECTOR, NATS_CONNECTOR, NEXMARK_CONNECTOR, + PULSAR_CONNECTOR, S3_CONNECTOR, S3_V2_CONNECTOR, }; use risingwave_pb::catalog::{ PbSchemaRegistryNameStrategy, PbSource, StreamSourceInfo, WatermarkDesc, @@ -99,11 +98,7 @@ async fn extract_avro_table_schema( info: &StreamSourceInfo, with_properties: &HashMap, ) -> Result> { - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Avro), - info, - with_properties, - )?; + let parser_config = SpecificParserConfig::new(info, with_properties)?; let conf = AvroParserConfig::new(parser_config.encoding_config).await?; let vec_column_desc = conf.map_to_columns()?; Ok(vec_column_desc @@ -120,11 +115,7 @@ async fn extract_upsert_avro_table_schema( info: &StreamSourceInfo, with_properties: &HashMap, ) -> Result<(Vec, Vec)> { - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Upsert, SourceEncode::Avro), - info, - with_properties, - )?; + let parser_config = SpecificParserConfig::new(info, with_properties)?; let conf = AvroParserConfig::new(parser_config.encoding_config).await?; let vec_column_desc = conf.map_to_columns()?; let mut vec_column_catalog = vec_column_desc @@ -164,11 +155,7 @@ async fn extract_debezium_avro_table_pk_columns( info: &StreamSourceInfo, with_properties: &HashMap, ) -> Result> { - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Debezium, SourceEncode::Avro), - info, - with_properties, - )?; + let parser_config = SpecificParserConfig::new(info, with_properties)?; let conf = DebeziumAvroParserConfig::new(parser_config.encoding_config).await?; Ok(conf.extract_pks()?.drain(..).map(|c| c.name).collect()) } @@ -178,11 +165,7 @@ async fn extract_debezium_avro_table_schema( info: &StreamSourceInfo, with_properties: &HashMap, ) -> Result> { - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Debezium, SourceEncode::Avro), - info, - with_properties, - )?; + let parser_config = SpecificParserConfig::new(info, with_properties)?; let conf = DebeziumAvroParserConfig::new(parser_config.encoding_config).await?; let vec_column_desc = conf.map_to_columns()?; let column_catalog = vec_column_desc @@ -204,13 +187,11 @@ async fn extract_protobuf_table_schema( proto_message_name: schema.message_name.0.clone(), row_schema_location: schema.row_schema_location.0.clone(), use_schema_registry: schema.use_schema_registry, + format: FormatType::Plain.into(), + row_encode: EncodeType::Protobuf.into(), ..Default::default() }; - let parser_config = SpecificParserConfig::new( - SourceStruct::new(SourceFormat::Plain, SourceEncode::Protobuf), - &info, - &with_properties, - )?; + let parser_config = SpecificParserConfig::new(&info, &with_properties)?; let conf = ProtobufParserConfig::new(parser_config.encoding_config).await?; let column_descs = conf.map_to_columns()?; diff --git a/src/source/src/source_desc.rs b/src/source/src/source_desc.rs index bf28c00e42f1f..161bbc41ceb63 100644 --- a/src/source/src/source_desc.rs +++ b/src/source/src/source_desc.rs @@ -18,15 +18,12 @@ use std::sync::Arc; use risingwave_common::catalog::ColumnDesc; use risingwave_common::error::ErrorCode::ProtocolError; use risingwave_common::error::{Result, RwError}; -use risingwave_connector::parser::SpecificParserConfig; +use risingwave_connector::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig}; use risingwave_connector::source::monitor::SourceMetrics; -use risingwave_connector::source::{ - ConnectorProperties, SourceColumnDesc, SourceColumnType, SourceEncode, SourceFormat, - SourceStruct, -}; +use risingwave_connector::source::{ConnectorProperties, SourceColumnDesc, SourceColumnType}; use risingwave_connector::ConnectorParams; use risingwave_pb::catalog::PbStreamSourceInfo; -use risingwave_pb::plan_common::{PbColumnCatalog, PbEncodeType, PbFormatType, RowFormatType}; +use risingwave_pb::plan_common::PbColumnCatalog; use crate::connector_source::ConnectorSource; use crate::fs_connector_source::FsConnectorSource; @@ -37,7 +34,6 @@ pub const DEFAULT_CONNECTOR_MESSAGE_BUFFER_SIZE: usize = 16; #[derive(Debug, Clone)] pub struct SourceDesc { pub source: ConnectorSource, - pub source_struct: SourceStruct, pub columns: Vec, pub metrics: Arc, @@ -48,7 +44,6 @@ pub struct SourceDesc { #[derive(Debug)] pub struct FsSourceDesc { pub source: FsConnectorSource, - pub source_struct: SourceStruct, pub columns: Vec, pub metrics: Arc, } @@ -107,9 +102,7 @@ impl SourceDescBuilder { pub fn build(mut self) -> Result { let columns = self.column_catalogs_to_source_column_descs(); - let source_struct = extract_source_struct(&self.source_info)?; - let psrser_config = - SpecificParserConfig::new(source_struct, &self.source_info, &self.properties)?; + let psrser_config = SpecificParserConfig::new(&self.source_info, &self.properties)?; let is_new_fs_source = ConnectorProperties::is_new_fs_connector_hash_map(&self.properties); if is_new_fs_source { @@ -126,7 +119,6 @@ impl SourceDescBuilder { Ok(SourceDesc { source, - source_struct, columns, metrics: self.metrics, is_new_fs_source, @@ -138,9 +130,16 @@ impl SourceDescBuilder { } pub fn build_fs_source_desc(&self) -> Result { - let source_struct = extract_source_struct(&self.source_info)?; - match (source_struct.format, source_struct.encode) { - (SourceFormat::Plain, SourceEncode::Csv | SourceEncode::Json) => {} + let parser_config = SpecificParserConfig::new(&self.source_info, &self.properties)?; + + match ( + &parser_config.protocol_config, + &parser_config.encoding_config, + ) { + ( + ProtocolProperties::Plain, + EncodingProperties::Csv(_) | EncodingProperties::Json(_), + ) => {} (format, encode) => { return Err(RwError::from(ProtocolError(format!( "Unsupported combination of format {:?} and encode {:?}", @@ -151,9 +150,6 @@ impl SourceDescBuilder { let columns = self.column_catalogs_to_source_column_descs(); - let parser_config = - SpecificParserConfig::new(source_struct, &self.source_info, &self.properties)?; - let source = FsConnectorSource::new( self.properties.clone(), columns.clone(), @@ -166,71 +162,12 @@ impl SourceDescBuilder { Ok(FsSourceDesc { source, - source_struct, columns, metrics: self.metrics.clone(), }) } } -// Only return valid (format, encode) -pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result { - // old version meta. - if let Ok(format) = info.get_row_format() { - let (format, encode) = match format { - RowFormatType::Json => (SourceFormat::Plain, SourceEncode::Json), - RowFormatType::Protobuf => (SourceFormat::Plain, SourceEncode::Protobuf), - RowFormatType::DebeziumJson => (SourceFormat::Debezium, SourceEncode::Json), - RowFormatType::Avro => (SourceFormat::Plain, SourceEncode::Avro), - RowFormatType::Maxwell => (SourceFormat::Maxwell, SourceEncode::Json), - RowFormatType::CanalJson => (SourceFormat::Canal, SourceEncode::Json), - RowFormatType::Csv => (SourceFormat::Plain, SourceEncode::Csv), - RowFormatType::Native => (SourceFormat::Native, SourceEncode::Native), - RowFormatType::DebeziumAvro => (SourceFormat::Debezium, SourceEncode::Avro), - RowFormatType::UpsertJson => (SourceFormat::Upsert, SourceEncode::Json), - RowFormatType::UpsertAvro => (SourceFormat::Upsert, SourceEncode::Avro), - RowFormatType::DebeziumMongoJson => (SourceFormat::DebeziumMongo, SourceEncode::Json), - RowFormatType::Bytes => (SourceFormat::Plain, SourceEncode::Bytes), - RowFormatType::RowUnspecified => unreachable!(), - }; - return Ok(SourceStruct::new(format, encode)); - } - let source_format = info.get_format()?; - let source_encode = info.get_row_encode()?; - let (format, encode) = match (source_format, source_encode) { - (PbFormatType::Plain, PbEncodeType::Json) => (SourceFormat::Plain, SourceEncode::Json), - (PbFormatType::Plain, PbEncodeType::Protobuf) => { - (SourceFormat::Plain, SourceEncode::Protobuf) - } - (PbFormatType::Debezium, PbEncodeType::Json) => { - (SourceFormat::Debezium, SourceEncode::Json) - } - (PbFormatType::Plain, PbEncodeType::Avro) => (SourceFormat::Plain, SourceEncode::Avro), - (PbFormatType::Maxwell, PbEncodeType::Json) => (SourceFormat::Maxwell, SourceEncode::Json), - (PbFormatType::Canal, PbEncodeType::Json) => (SourceFormat::Canal, SourceEncode::Json), - (PbFormatType::Plain, PbEncodeType::Csv) => (SourceFormat::Plain, SourceEncode::Csv), - (PbFormatType::Native, PbEncodeType::Native) => { - (SourceFormat::Native, SourceEncode::Native) - } - (PbFormatType::Debezium, PbEncodeType::Avro) => { - (SourceFormat::Debezium, SourceEncode::Avro) - } - (PbFormatType::Upsert, PbEncodeType::Json) => (SourceFormat::Upsert, SourceEncode::Json), - (PbFormatType::Upsert, PbEncodeType::Avro) => (SourceFormat::Upsert, SourceEncode::Avro), - (PbFormatType::DebeziumMongo, PbEncodeType::Json) => { - (SourceFormat::DebeziumMongo, SourceEncode::Json) - } - (PbFormatType::Plain, PbEncodeType::Bytes) => (SourceFormat::Plain, SourceEncode::Bytes), - (format, encode) => { - return Err(RwError::from(ProtocolError(format!( - "Unsupported combination of format {:?} and encode {:?}", - format, encode - )))); - } - }; - Ok(SourceStruct::new(format, encode)) -} - pub mod test_utils { use std::collections::HashMap;