feat: add the timer to track heartbeat intervel

GreptimeTeam · Dec 2, 2023 · bb5aa6d · bb5aa6d
1 parent 9c25008
commit bb5aa6d
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs
@@ -37,6 +37,7 @@ use crate::alive_keeper::RegionAliveKeeper;
 use crate::config::DatanodeOptions;
 use crate::error::{self, MetaClientInitSnafu, Result};
 use crate::event_listener::RegionServerEventReceiver;
+use crate::metrics;
 use crate::region_server::RegionServer;
 
 pub(crate) mod handler;
@@ -102,7 +103,13 @@ impl HeartbeatTask {
     ) -> Result<HeartbeatSender> {
         let client_id = meta_client.id();
 
+        let datanode = format!("datanode-{}", client_id.1);
         let (tx, mut rx) = meta_client.heartbeat().await.context(MetaClientInitSnafu)?;
+
+        let mut _last_received_lease = metrics::LAST_RECEIVED_HEARTBEAT_ELAPSED
+            .with_label_values(&[&datanode])
+            .start_timer();
+
         let _handle = common_runtime::spawn_bg(async move {
             while let Some(res) = match rx.message().await {
                 Ok(m) => m,
@@ -114,6 +121,12 @@ impl HeartbeatTask {
                 if let Some(msg) = res.mailbox_message.as_ref() {
                     info!("Received mailbox message: {msg:?}, meta_client id: {client_id:?}");
                 }
+                if res.region_lease.as_ref().is_some() {
+                    // Resets the timer.
+                    _last_received_lease = metrics::LAST_RECEIVED_HEARTBEAT_ELAPSED
+                        .with_label_values(&[&datanode])
+                        .start_timer();
+                }
                 let ctx = HeartbeatResponseHandlerContext::new(mailbox.clone(), res);
                 if let Err(e) = Self::handle_response(ctx, handler_executor.clone()).await {
                     error!(e; "Error while handling heartbeat response");

diff --git a/src/datanode/src/metrics.rs b/src/datanode/src/metrics.rs
@@ -26,4 +26,10 @@ lazy_static! {
         &[REGION_REQUEST_TYPE]
     )
     .unwrap();
+    pub static ref LAST_RECEIVED_HEARTBEAT_ELAPSED: HistogramVec = register_histogram_vec!(
+        "last_received_heartbeat_lease_elapsed",
+        "last received heartbeat lease elapsed",
+        &["datanode"]
+    )
+    .unwrap();
 }