feat: add a metric to track region lease expired

GreptimeTeam · Dec 4, 2023 · 4d43996 · 4d43996
1 parent f052d99
commit 4d43996
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 1 deletion.
diff --git a/src/datanode/src/alive_keeper.rs b/src/datanode/src/alive_keeper.rs
@@ -37,6 +37,7 @@ use tokio::time::{Duration, Instant};
 
 use crate::error::{self, Result};
 use crate::event_listener::{RegionServerEvent, RegionServerEventReceiver};
+use crate::metrics;
 use crate::region_server::RegionServer;
 
 /// [RegionAliveKeeper] manages all [CountdownTaskHandle]s.
@@ -339,6 +340,10 @@ impl Drop for CountdownTaskHandle {
             "Aborting region alive countdown task for region {}",
             self.region_id
         );
+        // Resets the metric if the CountdownTask quits.
+        metrics::LEASE_EXPIRED_REGION
+            .with_label_values(&[&format!("{}", self.region_id)])
+            .set(0);
         self.handler.abort();
     }
 }
@@ -359,7 +364,6 @@ impl CountdownTask {
         // "start countdown" command will be sent from heartbeat task).
         let countdown = tokio::time::sleep_until(far_future);
         tokio::pin!(countdown);
-
         let region_id = self.region_id;
         loop {
             tokio::select! {
@@ -407,6 +411,9 @@ impl CountdownTask {
                 () = &mut countdown => {
                     info!("The region lease expired, set region {region_id} to readonly.");
                     let _ = self.region_server.set_writable(self.region_id, false);
+                    metrics::LEASE_EXPIRED_REGION
+                        .with_label_values(&[&format!("{}", self.region_id)])
+                        .add(1);
                     // resets the countdown.
                     let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
                     countdown.as_mut().reset(far_future);

diff --git a/src/datanode/src/metrics.rs b/src/datanode/src/metrics.rs
@@ -19,6 +19,7 @@ use prometheus::*;
 pub const REGION_REQUEST_TYPE: &str = "datanode_region_request_type";
 
 pub const REGION_ROLE: &str = "region_role";
+pub const REGION_ID: &str = "region_id";
 
 lazy_static! {
     /// The elapsed time of handling a request in the region_server.
@@ -34,6 +35,12 @@ lazy_static! {
         "last received heartbeat lease elapsed",
     )
     .unwrap();
+    pub static ref LEASE_EXPIRED_REGION: IntGaugeVec = register_int_gauge_vec!(
+        "lease_expired_region",
+        "lease expired region",
+        &[REGION_ID]
+    )
+    .unwrap();
     /// The received region leases via heartbeat.
     pub static ref HEARTBEAT_REGION_LEASES: IntGaugeVec = register_int_gauge_vec!(
         "heartbeat_region_leases",