add retry metrics

risingwavelabs · Mar 7, 2024 · efe51b9 · efe51b9
1 parent 3774877
commit efe51b9
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 3 deletions.
diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json
diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py
@@ -4068,6 +4068,10 @@ def section_udf(outer_panels):
                             f"sum(rate({metric('udf_failure_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL})",
                             "udf_failure_count - {{%s}}" % NODE_LABEL,
                         ),
+                        panels.target(
+                            f"sum(rate({metric('udf_retry_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "udf_retry_count - {{%s}}" % NODE_LABEL,
+                        ),
                         panels.target(
                             f"sum(rate({metric('udf_success_count')}[$__rate_interval])) by (link, name, fragment_id)",
                             "udf_success_count - {{link}} {{name}} {{fragment_id}}",
@@ -4076,6 +4080,10 @@ def section_udf(outer_panels):
                             f"sum(rate({metric('udf_failure_count')}[$__rate_interval])) by (link, name, fragment_id)",
                             "udf_failure_count - {{link}} {{name}} {{fragment_id}}",
                         ),
+                        panels.target(
+                            f"sum(rate({metric('udf_retry_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "udf_retry_count - {{%s}}" % NODE_LABEL,
+                        ),
                     ],
                 ),
                 panels.timeseries_count(

diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json
diff --git a/src/expr/udf/src/external.rs b/src/expr/udf/src/external.rs
@@ -200,7 +200,7 @@ impl ArrowFlightUdfClient {
                     return ret;
                 }
             }
-            metrics.udf_failure_count.with_label_values(labels).inc();
+            metrics.udf_retry_count.with_label_values(labels).inc();
             tokio::time::sleep(backoff).await;
             backoff *= 2;
         }

diff --git a/src/expr/udf/src/metrics.rs b/src/expr/udf/src/metrics.rs
@@ -27,6 +27,8 @@ pub struct Metrics {
     pub udf_success_count: IntCounterVec,
     /// Number of failed UDF calls.
     pub udf_failure_count: IntCounterVec,
+    /// Total number of retried UDF calls.
+    pub udf_retry_count: IntCounterVec,
     /// Input chunk rows of UDF calls.
     pub udf_input_chunk_rows: HistogramVec,
     /// The latency of UDF calls in seconds.
@@ -58,6 +60,13 @@ impl Metrics {
             registry
         )
         .unwrap();
+        let udf_retry_count = register_int_counter_vec_with_registry!(
+            "udf_retry_count",
+            "Total number of retried UDF calls",
+            labels,
+            registry
+        )
+        .unwrap();
         let udf_input_chunk_rows = register_histogram_vec_with_registry!(
             "udf_input_chunk_rows",
             "Input chunk rows of UDF calls",
@@ -92,6 +101,7 @@ impl Metrics {
         Metrics {
             udf_success_count,
             udf_failure_count,
+            udf_retry_count,
             udf_input_chunk_rows,
             udf_latency,
             udf_input_rows,