fix: stale metrics of meta leader election (#16334)

risingwavelabs · Apr 16, 2024 · dba4fab · dba4fab
1 parent 18de4d1
commit dba4fab
Showing 1 changed file with 12 additions and 8 deletions.
diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs
@@ -47,17 +47,17 @@ use crate::rpc::ElectionClientRef;
 
 #[derive(Clone)]
 pub struct MetaMetrics {
-    /// ********************************** Meta ************************************
+    // ********************************** Meta ************************************
     /// The number of workers in the cluster.
     pub worker_num: IntGaugeVec,
     /// The roles of all meta nodes in the cluster.
     pub meta_type: IntGaugeVec,
 
-    /// ********************************** gRPC ************************************
+    // ********************************** gRPC ************************************
     /// gRPC latency of meta services
     pub grpc_latency: HistogramVec,
 
-    /// ********************************** Barrier ************************************
+    // ********************************** Barrier ************************************
     /// The duration from barrier injection to commit
     /// It is the sum of inflight-latency, sync-latency and wait-commit-latency
     pub barrier_latency: Histogram,
@@ -73,11 +73,11 @@ pub struct MetaMetrics {
     /// The timestamp (UNIX epoch seconds) of the last committed barrier's epoch time.
     pub last_committed_barrier_time: IntGauge,
 
-    /// ********************************** Recovery ************************************
+    // ********************************** Recovery ************************************
     pub recovery_failure_cnt: IntCounter,
     pub recovery_latency: Histogram,
 
-    /// ********************************** Hummock ************************************
+    // ********************************** Hummock ************************************
     /// Max committed epoch
     pub max_committed_epoch: IntGauge,
     /// The smallest epoch that has not been `GCed`.
@@ -160,16 +160,16 @@ pub struct MetaMetrics {
     pub compaction_event_consumed_latency: Histogram,
     pub compaction_event_loop_iteration_latency: Histogram,
 
-    /// ********************************** Object Store ************************************
+    // ********************************** Object Store ************************************
     // Object store related metrics (for backup/restore and version checkpoint)
     pub object_store_metric: Arc<ObjectStoreMetrics>,
 
-    /// ********************************** Source ************************************
+    // ********************************** Source ************************************
     /// supervisor for which source is still up.
     pub source_is_up: LabelGuardedIntGaugeVec<2>,
     pub source_enumerator_metrics: Arc<SourceEnumeratorMetrics>,
 
-    /// ********************************** Fragment ************************************
+    // ********************************** Fragment ************************************
     /// A dummpy gauge metrics with its label to be the mapping from actor id to fragment id
     pub actor_info: IntGaugeVec,
     /// A dummpy gauge metrics with its label to be the mapping from table id to actor id
@@ -744,6 +744,10 @@ pub fn start_worker_info_monitor(
                 }
             };
 
+            // Reset metrics to clean the stale labels e.g. invalid lease ids
+            meta_metrics.worker_num.reset();
+            meta_metrics.meta_type.reset();
+
             for (worker_type, worker_num) in node_map {
                 meta_metrics
                     .worker_num