feat: add metrics for lease keeper

src/meta-srv/src/metrics.rs

@@ -49,4 +49,20 @@ lazy_static! {
         "meta kv cache batch get hit rate"
     )
     .unwrap();
+    /// The number of non-existing regions detected by lease keeper.
+    pub static ref METRIC_META_LEASE_KEEPER_NON_EXISTING_REGION_NUM: IntGaugeVec =
+        register_int_gauge_vec!(
+            "meta_lease_keeper_non_existing_region_num",
+            "meta lease keeper non-existing region num",
+            &["datanode"]
+        )
+        .unwrap();
+    /// The number of region roles changes detected by the lease keeper.
+    pub static ref METRIC_META_LEASE_KEEPER_REGION_ROLE_CHANGE_NUM: IntGaugeVec =
+        register_int_gauge_vec!(
+            "meta_lease_keeper_region_role_change_num",
+            "meta lease keeper region role change num",
+            &["datanode","region_id","op"]
+        )
+        .unwrap();
 }

src/meta-srv/src/region/lease_keeper.rs

@@ -24,6 +24,7 @@ use store_api::region_engine::RegionRole;
 use store_api::storage::{RegionId, TableId};
 
 use crate::error::{self, Result};
+use crate::metrics;
 
 pub type RegionLeaseKeeperRef = Arc<RegionLeaseKeeper>;
 
@@ -143,7 +144,7 @@ impl RegionLeaseKeeper {
     /// Otherwise the lease of regions will't be renewed.
     pub async fn renew_region_leases(
         &self,
-        _cluster_id: u64,
+        cluster_id: u64,
         datanode_id: DatanodeId,
         regions: &[(RegionId, RegionRole)],
     ) -> Result<RenewRegionLeasesResponse> {
@@ -158,12 +159,25 @@ impl RegionLeaseKeeper {
         let mut renewed = HashMap::new();
         let mut non_exists = HashSet::new();
 
+        let datanode_ident = format!("{}-{}", cluster_id, datanode_id);
+
         for &(region, role) in regions {
             match self.renew_region_lease(&table_metadata, datanode_id, region, role) {
-                Some((region, role)) => {
-                    let _ = renewed.insert(region, role);
+                Some((region, renewed_role)) => {
+                    let _ = renewed.insert(region, renewed_role);
+                    // Metrics for role changes.
+                    if renewed_role != role {
+                        let region_id = format!("{region}");
+                        let op = format!("{role} -> {renewed_role}");
+                        metrics::METRIC_META_LEASE_KEEPER_REGION_ROLE_CHANGE_NUM
+                            .with_label_values(&[&datanode_ident, &region_id, &op])
+                            .inc()
+                    }
                 }
                 None => {
+                    metrics::METRIC_META_LEASE_KEEPER_NON_EXISTING_REGION_NUM
+                        .with_label_values(&[&datanode_ident])
+                        .inc();
                     let _ = non_exists.insert(region);
                 }
             }

src/store-api/src/region_engine.rs

@@ -14,6 +14,7 @@
 
 //! Region Engine's definition
 
+use std::fmt::Display;
 use std::sync::Arc;
 
 use api::greptime_proto::v1::meta::{GrantedRegion as PbGrantedRegion, RegionRole as PbRegionRole};
@@ -85,6 +86,15 @@ pub enum RegionRole {
     Leader,
 }
 
+impl Display for RegionRole {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RegionRole::Follower => write!(f, "Follower"),
+            RegionRole::Leader => write!(f, "Leader"),
+        }
+    }
+}
+
 impl RegionRole {
     pub fn writable(&self) -> bool {
         matches!(self, RegionRole::Leader)