fix(dashboard): calculate lru_evicted_watermark_time_diff_ms in proms…

…ql (#12431)
risingwavelabs · Sep 21, 2023 · f075a6b · f075a6b
1 parent a7b7688
commit f075a6b
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 18 deletions.
diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json
diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py
@@ -3216,7 +3216,7 @@ def section_memory_manager(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"{metric('lru_evicted_watermark_time_diff_ms')}",
+                            f"{metric('lru_current_watermark_time_ms')} - on() group_right() {metric('lru_evicted_watermark_time_ms')}",
                             "table {{table_id}} actor {{actor_id}} desc: {{desc}}",
                         ),
                     ],

diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json
diff --git a/src/stream/src/cache/managed_lru.rs b/src/stream/src/cache/managed_lru.rs
@@ -41,7 +41,7 @@ pub struct ManagedLruCache<K, V, S = DefaultHasher, A: Clone + Allocator = Globa
     /// The metrics of memory usage
     memory_usage_metrics: IntGauge,
     // The metrics of evicted watermark time
-    lru_evicted_watermark_time_diff_ms: IntGauge,
+    lru_evicted_watermark_time_ms: IntGauge,
     // Metrics info
     metrics_info: MetricsInfo,
     /// The size reported last time
@@ -65,11 +65,11 @@ impl<K, V, S, A: Clone + Allocator> Drop for ManagedLruCache<K, V, S, A> {
         };
         if let Err(e) = info
             .metrics
-            .lru_evicted_watermark_time_diff_ms
+            .lru_evicted_watermark_time_ms
             .remove_label_values(&[&info.table_id, &info.actor_id, &info.desc])
         {
             warn!(
-                "unable to remove lru_evicted_watermark_time_diff_ms of {} {} {}: {:?}",
+                "unable to remove lru_evicted_watermark_time_ms of {} {} {}: {:?}",
                 info.table_id, info.actor_id, info.desc, e
             );
         }
@@ -94,22 +94,21 @@ impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Al
             ]);
         memory_usage_metrics.set(0.into());
 
-        let lru_evicted_watermark_time_diff_ms = metrics_info
+        let lru_evicted_watermark_time_ms = metrics_info
             .metrics
-            .lru_evicted_watermark_time_diff_ms
+            .lru_evicted_watermark_time_ms
             .with_label_values(&[
                 &metrics_info.table_id,
                 &metrics_info.actor_id,
                 &metrics_info.desc,
             ]);
-        lru_evicted_watermark_time_diff_ms.set(watermark_epoch.load(Ordering::Relaxed) as _);
 
         Self {
             inner,
             watermark_epoch,
             kv_heap_size: 0,
             memory_usage_metrics,
-            lru_evicted_watermark_time_diff_ms,
+            lru_evicted_watermark_time_ms,
             metrics_info,
             last_reported_size_bytes: 0,
         }
@@ -244,9 +243,8 @@ impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Al
     }
 
     fn report_evicted_watermark_time(&self, epoch: u64) {
-        self.lru_evicted_watermark_time_diff_ms.set(
-            (Epoch(self.load_cur_epoch()).physical_time() - Epoch(epoch).physical_time()) as _,
-        );
+        self.lru_evicted_watermark_time_ms
+            .set(Epoch(epoch).physical_time() as _);
     }
 
     fn load_cur_epoch(&self) -> u64 {

diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs
@@ -127,7 +127,7 @@ pub struct StreamingMetrics {
     pub lru_physical_now_ms: IntGauge,
     pub lru_runtime_loop_count: IntCounter,
     pub lru_watermark_step: IntGauge,
-    pub lru_evicted_watermark_time_diff_ms: GenericGaugeVec<AtomicI64>,
+    pub lru_evicted_watermark_time_ms: GenericGaugeVec<AtomicI64>,
     pub jemalloc_allocated_bytes: IntGauge,
     pub jemalloc_active_bytes: IntGauge,
 
@@ -725,9 +725,9 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let lru_evicted_watermark_time_diff_ms = register_int_gauge_vec_with_registry!(
-            "lru_evicted_watermark_time_diff_ms",
-            "The diff between current watermark and latest evicted watermark time by actors",
+        let lru_evicted_watermark_time_ms = register_int_gauge_vec_with_registry!(
+            "lru_evicted_watermark_time_ms",
+            "The latest evicted watermark time by actors",
             &["table_id", "actor_id", "desc"],
             registry
         )
@@ -864,7 +864,7 @@ impl StreamingMetrics {
             lru_physical_now_ms,
             lru_runtime_loop_count,
             lru_watermark_step,
-            lru_evicted_watermark_time_diff_ms,
+            lru_evicted_watermark_time_ms,
             jemalloc_allocated_bytes,
             jemalloc_active_bytes,
             user_compute_error_count,