diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs index 493c2bcc7462..649230ccd2e3 100644 --- a/src/meta-srv/src/metrics.rs +++ b/src/meta-srv/src/metrics.rs @@ -16,37 +16,45 @@ use lazy_static::lazy_static; use prometheus::*; lazy_static! { - pub static ref METRIC_META_KV_REQUEST: HistogramVec = register_histogram_vec!( - "meta_kv_request", + /// Elapsed time to responding kv requests. + pub static ref METRIC_META_KV_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!( + "meta_kv_request_elapsed", "meta kv request", &["target", "op", "cluster_id"] ) .unwrap(); + /// The heartbeat connection gauge. pub static ref METRIC_META_HEARTBEAT_CONNECTION_NUM: IntGauge = register_int_gauge!( "meta_heartbeat_connection_num", "meta heartbeat connection num" ) .unwrap(); + /// Elapsed time to execution of heartbeat handlers. pub static ref METRIC_META_HANDLER_EXECUTE: HistogramVec = register_histogram_vec!("meta_handler_execute", "meta handler execute", &["name"]).unwrap(); + /// Inactive region gauge. pub static ref METRIC_META_INACTIVE_REGIONS: IntGauge = register_int_gauge!("meta_inactive_regions", "meta inactive regions").unwrap(); - pub static ref METRIC_META_LEADER_CACHED_KV_LOAD: HistogramVec = + /// Elapsed time to leader cache kv. + pub static ref METRIC_META_LEADER_CACHED_KV_LOAD_ELAPSED: HistogramVec = register_histogram_vec!("meta_leader_cache_kv_load", "meta load cache", &["prefix"]) .unwrap(); - pub static ref METRIC_META_LOAD_FOLLOWER_METADATA: Histogram = register_histogram!( - "meta_load_follower_metadata", + /// Elapsed time to load follower region metadata. + pub static ref METRIC_META_LOAD_FOLLOWER_METADATA_ELAPSED: Histogram = register_histogram!( + "meta_load_follower_metadata_elapsed", "meta load follower regions metadata elapsed" ) .unwrap(); - pub static ref METRIC_META_LOAD_LEADER_METADATA: Histogram = register_histogram!( - "meta_load_leader_metadata", + /// Elapsed time to load leader region metadata. + pub static ref METRIC_META_LOAD_LEADER_METADATA_ELAPSED: Histogram = register_histogram!( + "meta_load_leader_metadata_elapsed", "meta load leader regions metadata elapsed" ) .unwrap(); - pub static ref METRIC_META_KV_CACHE_BATCH_GET_HIT_RATE: Gauge = register_gauge!( - "meta_kv_cache_batch_get_hit_rate", - "meta kv cache batch get hit rate" - ) - .unwrap(); + /// Meta kv cache hit counter. + pub static ref METRIC_META_KV_CACHE_HIT: IntCounterVec = + register_int_counter_vec!("meta_kv_cache_hit", "meta kv cache hit", &["op"]).unwrap(); + /// Meta kv cache miss counter. + pub static ref METRIC_META_KV_CACHE_MISS: IntCounterVec = + register_int_counter_vec!("meta_kv_cache_miss", "meta kv cache miss", &["op"]).unwrap(); } diff --git a/src/meta-srv/src/region/lease_keeper.rs b/src/meta-srv/src/region/lease_keeper.rs index f29495e1e5f8..449b6063cd3e 100644 --- a/src/meta-srv/src/region/lease_keeper.rs +++ b/src/meta-srv/src/region/lease_keeper.rs @@ -93,7 +93,7 @@ impl RegionLeaseKeeper { let table_ids = tables.keys().copied().collect::>(); let metadata_subset = { - let _timer = metrics::METRIC_META_LOAD_LEADER_METADATA.start_timer(); + let _timer = metrics::METRIC_META_LOAD_LEADER_METADATA_ELAPSED.start_timer(); self.collect_tables_metadata(&table_ids).await? }; @@ -140,7 +140,7 @@ impl RegionLeaseKeeper { let table_ids = tables.keys().copied().collect::>(); let metadata_subset = { - let _timer = metrics::METRIC_META_LOAD_FOLLOWER_METADATA.start_timer(); + let _timer = metrics::METRIC_META_LOAD_FOLLOWER_METADATA_ELAPSED.start_timer(); self.collect_tables_metadata(&table_ids).await? }; diff --git a/src/meta-srv/src/service/store.rs b/src/meta-srv/src/service/store.rs index dfee0bc58e8d..1a291ada1f25 100644 --- a/src/meta-srv/src/service/store.rs +++ b/src/meta-srv/src/service/store.rs @@ -33,7 +33,7 @@ use tonic::{Request, Response}; use crate::error::{self, MissingRequestHeaderSnafu}; use crate::metasrv::MetaSrv; -use crate::metrics::METRIC_META_KV_REQUEST; +use crate::metrics::METRIC_META_KV_REQUEST_ELAPSED; use crate::service::GrpcResult; #[async_trait::async_trait] @@ -48,7 +48,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[self.kv_backend().name(), "range", cluster_id_str.as_str()]) .start_timer(); @@ -74,7 +74,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[self.kv_backend().name(), "put", cluster_id_str.as_str()]) .start_timer(); @@ -100,7 +100,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[ self.kv_backend().name(), "batch_get", @@ -130,7 +130,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[ self.kv_backend().name(), "batch_pub", @@ -163,7 +163,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[ self.kv_backend().name(), "batch_delete", @@ -196,7 +196,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[ self.kv_backend().name(), "compare_and_put", @@ -229,7 +229,7 @@ impl store_server::Store for MetaSrv { .cluster_id; let cluster_id_str = cluster_id.to_string(); - let _timer = METRIC_META_KV_REQUEST + let _timer = METRIC_META_KV_REQUEST_ELAPSED .with_label_values(&[ self.kv_backend().name(), "delete_range", diff --git a/src/meta-srv/src/service/store/cached_kv.rs b/src/meta-srv/src/service/store/cached_kv.rs index a1d251971c29..af9e0a5cecac 100644 --- a/src/meta-srv/src/service/store/cached_kv.rs +++ b/src/meta-srv/src/service/store/cached_kv.rs @@ -94,7 +94,8 @@ impl LeaderCachedKvBackend { /// The caller MUST ensure during the loading, there are no mutation requests reaching the `LeaderCachedKvStore`. pub async fn load(&self) -> Result<()> { for prefix in &CACHE_KEY_PREFIXES[..] { - let _timer = metrics::METRIC_META_LEADER_CACHED_KV_LOAD.with_label_values(&[prefix]); + let _timer = + metrics::METRIC_META_LEADER_CACHED_KV_LOAD_ELAPSED.with_label_values(&[prefix]); // TODO(weny): Refactors PaginationStream's output to unary output. let stream = PaginationStream::new( @@ -261,8 +262,9 @@ impl KvBackend for LeaderCachedKvBackend { .map(|kv| kv.key.clone()) .collect::>(); - let hit_rate = hit_keys.len() as f64 / req.keys.len() as f64; - metrics::METRIC_META_KV_CACHE_BATCH_GET_HIT_RATE.set(hit_rate); + metrics::METRIC_META_KV_CACHE_HIT + .with_label_values(&[&"batch_get"]) + .inc_by(hit_keys.len() as u64); let missed_keys = req .keys @@ -270,6 +272,10 @@ impl KvBackend for LeaderCachedKvBackend { .filter(|key| !hit_keys.contains(*key)) .cloned() .collect::>(); + metrics::METRIC_META_KV_CACHE_MISS + .with_label_values(&[&"batch_get"]) + .inc_by(missed_keys.len() as u64); + let remote_req = BatchGetRequest { keys: missed_keys }; let ver = self.get_version();