diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs index 493c2bcc7462..ee4083eac551 100644 --- a/src/meta-srv/src/metrics.rs +++ b/src/meta-srv/src/metrics.rs @@ -49,4 +49,20 @@ lazy_static! { "meta kv cache batch get hit rate" ) .unwrap(); + /// The number of non-existing regions detected by lease keeper. + pub static ref METRIC_META_LEASE_KEEPER_NON_EXISTING_REGION_NUM: IntGaugeVec = + register_int_gauge_vec!( + "meta_lease_keeper_non_existing_region_num", + "meta lease keeper non-existing region num", + &["datanode"] + ) + .unwrap(); + /// The number of region roles changes detected by the lease keeper. + pub static ref METRIC_META_LEASE_KEEPER_REGION_ROLE_CHANGE_NUM: IntGaugeVec = + register_int_gauge_vec!( + "meta_lease_keeper_region_role_change_num", + "meta lease keeper region role change num", + &["datanode","region_id","op"] + ) + .unwrap(); } diff --git a/src/meta-srv/src/region/lease_keeper.rs b/src/meta-srv/src/region/lease_keeper.rs index fb8c3901d084..c69a10ae742b 100644 --- a/src/meta-srv/src/region/lease_keeper.rs +++ b/src/meta-srv/src/region/lease_keeper.rs @@ -24,6 +24,7 @@ use store_api::region_engine::RegionRole; use store_api::storage::{RegionId, TableId}; use crate::error::{self, Result}; +use crate::metrics; pub type RegionLeaseKeeperRef = Arc; @@ -143,7 +144,7 @@ impl RegionLeaseKeeper { /// Otherwise the lease of regions will't be renewed. pub async fn renew_region_leases( &self, - _cluster_id: u64, + cluster_id: u64, datanode_id: DatanodeId, regions: &[(RegionId, RegionRole)], ) -> Result { @@ -158,12 +159,25 @@ impl RegionLeaseKeeper { let mut renewed = HashMap::new(); let mut non_exists = HashSet::new(); + let datanode_ident = format!("{}-{}", cluster_id, datanode_id); + for &(region, role) in regions { match self.renew_region_lease(&table_metadata, datanode_id, region, role) { - Some((region, role)) => { - let _ = renewed.insert(region, role); + Some((region, renewed_role)) => { + let _ = renewed.insert(region, renewed_role); + // Metrics for role changes. + if renewed_role != role { + let region_id = format!("{region}"); + let op = format!("{role} -> {renewed_role}"); + metrics::METRIC_META_LEASE_KEEPER_REGION_ROLE_CHANGE_NUM + .with_label_values(&[&datanode_ident, ®ion_id, &op]) + .inc() + } } None => { + metrics::METRIC_META_LEASE_KEEPER_NON_EXISTING_REGION_NUM + .with_label_values(&[&datanode_ident]) + .inc(); let _ = non_exists.insert(region); } } diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index 27c0d7a93a6c..5bd03f26aeda 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -14,6 +14,7 @@ //! Region Engine's definition +use std::fmt::Display; use std::sync::Arc; use api::greptime_proto::v1::meta::{GrantedRegion as PbGrantedRegion, RegionRole as PbRegionRole}; @@ -85,6 +86,15 @@ pub enum RegionRole { Leader, } +impl Display for RegionRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RegionRole::Follower => write!(f, "Follower"), + RegionRole::Leader => write!(f, "Leader"), + } + } +} + impl RegionRole { pub fn writable(&self) -> bool { matches!(self, RegionRole::Leader)