feat: track prometheus HTTP API's query latency (#4458)

* feat: track prometheus HTTP API's query latency Signed-off-by: Ruihang Xia <[email protected]> * update grafana config Signed-off-by: Ruihang Xia <[email protected]> * chore: Update src/servers/src/metrics.rs --------- Signed-off-by: Ruihang Xia <[email protected]> Co-authored-by: Yingwen <[email protected]>
GreptimeTeam · Jul 29, 2024 · 1895a54 · 1895a54
1 parent f0c953f
commit 1895a54
Show file tree

Hide file tree

Showing 3 changed files with 158 additions and 5 deletions.
diff --git a/grafana/greptimedb.json b/grafana/greptimedb.json
@@ -679,7 +679,7 @@
           "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))",
           "hide": false,
           "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "sql-{{db}}-p95",
           "range": true,
           "refId": "SQL P95"
         },
@@ -692,7 +692,7 @@
           "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))",
           "hide": false,
           "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "sql-{{db}}-p99",
           "range": true,
           "refId": "SQL P99"
         },
@@ -705,7 +705,7 @@
           "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))",
           "hide": false,
           "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "promstore-read-{{db}}-p95",
           "range": true,
           "refId": "PromStore Read P95"
         },
@@ -718,9 +718,35 @@
           "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))",
           "hide": false,
           "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "promstore-read-{{db}}-p99",
           "range": true,
           "refId": "PromStore Read P99"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-1}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "prom-promql-{{db}}-{{method}}-p95",
+          "range": true,
+          "refId": "Prometheus PromQL P95"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-1}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "prom-promql-{{db}}-{{method}}-p99",
+          "range": true,
+          "refId": "Prometheus PromQL P99"
         }
       ],
       "title": "HTTP query elapsed",
@@ -1090,6 +1116,106 @@
       "title": "gRPC insert elapsed",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS-1}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 36,
+      "interval": "1s",
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-1}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "sum by(db) (rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{db}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Logs ingest rate (number of lines)",
+      "type": "timeseries"
+    },
     {
       "collapsed": false,
       "gridPos": {
@@ -2626,4 +2752,4 @@
   "uid": "e7097237-669b-4f8d-b751-13067afbfb68",
   "version": 14,
   "weekStart": ""
-}
+}
diff --git a/src/servers/src/http/prometheus.rs b/src/servers/src/http/prometheus.rs
@@ -193,6 +193,10 @@ pub async fn instant_query(
         query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
     }
 
+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "instant_query"])
+        .start_timer();
+
     let result = handler.do_query(&prom_query, query_ctx).await;
     let (metric_name, result_type) = match retrieve_metric_name_and_result_type(&prom_query.query) {
         Ok((metric_name, result_type)) => (metric_name.unwrap_or_default(), result_type),
@@ -242,6 +246,10 @@ pub async fn range_query(
         query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
     }
 
+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "range_query"])
+        .start_timer();
+
     let result = handler.do_query(&prom_query, query_ctx).await;
     let metric_name = match retrieve_metric_name_and_result_type(&prom_query.query) {
         Err(err) => {
@@ -316,6 +324,10 @@ pub async fn labels_query(
         queries = form_params.matches.0;
     }
 
+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "labels_query"])
+        .start_timer();
+
     // Fetch all tag columns. It will be used as white-list for tag names.
     let mut labels = match get_all_column_names(&catalog, &schema, &handler.catalog_manager()).await
     {
@@ -687,6 +699,10 @@ pub async fn label_values_query(
     let (catalog, schema) = get_catalog_schema(&params.db, &query_ctx);
     let query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
 
+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "label_values_query"])
+        .start_timer();
+
     if label_name == METRIC_NAME_LABEL {
         let mut table_names = match handler
             .catalog_manager()
@@ -968,6 +984,10 @@ pub async fn series_query(
         query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
     }
 
+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "series_query"])
+        .start_timer();
+
     let mut series = Vec::new();
     let mut merge_map = HashMap::new();
     for query in queries {

diff --git a/src/servers/src/metrics.rs b/src/servers/src/metrics.rs
@@ -120,6 +120,13 @@ lazy_static! {
         &[METRIC_DB_LABEL]
     )
     .unwrap();
+    /// Http prometheus endpoint query duration per database.
+    pub static ref METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED: HistogramVec = register_histogram_vec!(
+        "greptime_servers_http_prometheus_promql_elapsed",
+        "servers http prometheus promql elapsed",
+        &[METRIC_DB_LABEL, METRIC_METHOD_LABEL]
+    )
+    .unwrap();
     pub static ref METRIC_HTTP_OPENTELEMETRY_METRICS_ELAPSED: HistogramVec =
         register_histogram_vec!(
             "greptime_servers_http_otlp_metrics_elapsed",