add query frontend metrics config to tempo distributed helm chart

Signed-off-by: Farid Mammadov <[email protected]>
grafana · Nov 21, 2024 · 2d5cbfe · 2d5cbfe
1 parent 8103201
commit 2d5cbfe
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 3 deletions.
diff --git a/charts/tempo-distributed/Chart.yaml b/charts/tempo-distributed/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: tempo-distributed
 description: Grafana Tempo in MicroService mode
 type: application
-version: 1.22.2
+version: 1.22.3
 appVersion: 2.6.0
 engine: gotpl
 home: https://grafana.com/docs/tempo/latest/

diff --git a/charts/tempo-distributed/README.md b/charts/tempo-distributed/README.md
@@ -752,7 +752,13 @@ The memcached default args are removed and should be provided manually. The sett
 | queryFrontend.autoscaling.targetMemoryUtilizationPercentage | string | `nil` | Target memory utilisation percentage for the query-frontend |
 | queryFrontend.config.max_outstanding_per_tenant | int | `2000` | Maximum number of outstanding requests per tenant per frontend; requests beyond this error with HTTP 429. |
 | queryFrontend.config.max_retries | int | `2` | Number of times to retry a request sent to a querier |
-| queryFrontend.config.metrics.max_duration | string | `"3h"` |  |
+| queryFrontend.config.metrics.concurrent_jobs | int | `1000` | The number of concurrent jobs to execute when querying the backend. |
+| queryFrontend.config.metrics.target_bytes_per_job | int | `104857600` | The target number of bytes for each job to handle when querying the backend. |
+| queryFrontend.config.metrics.max_duration | string | `3h` | The maximum allowed time range for a metrics query. 0 disables this limit. |
+| queryFrontend.config.metrics.query_backend_after | string | `30m` | Controls where the query-frontend searches for traces. Time ranges older than query_backend_after will be searched in the backend/object storage only. Time ranges between query_backend_after and now will be queried from the metrics-generators. |
+| queryFrontend.config.metrics.interval | string | `5m` | The target length of time for each job to handle when querying the backend. |
+| queryFrontend.config.metrics.duration_slo | string | `0s` | If set to a non-zero value, it's value will be used to decide if query is within SLO or not. Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data. NOTE: `duration_slo` and `throughput_bytes_slo` both must be configured for it to work |
+| queryFrontend.config.metrics.throughput_bytes_slo | int | `0` | If set to a non-zero value, it's value will be used to decide if query is within SLO or not. Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data. |
 | queryFrontend.config.search.concurrent_jobs | int | `1000` | The number of concurrent jobs to execute when searching the backend |
 | queryFrontend.config.search.target_bytes_per_job | int | `104857600` | The target number of bytes for each job to handle when performing a backend search |
 | queryFrontend.config.trace_by_id | object | `{"query_shards":50}` | Trace by ID lookup configuration |

diff --git a/charts/tempo-distributed/values.yaml b/charts/tempo-distributed/values.yaml
@@ -837,9 +837,26 @@ queryFrontend:
       # -- The number of shards to split a trace by id query into.
       query_shards: 50
     metrics:
+      # The number of concurrent jobs to execute when querying the backend.
+      concurrent_jobs: 1000
+      # The target number of bytes for each job to handle when querying the backend.
+      target_bytes_per_job: 104857600
       # The maximum allowed time range for a metrics query.
       # 0 disables this limit.
       max_duration: 3h
+      # query_backend_after controls where the query-frontend searches for traces.
+      # Time ranges older than query_backend_after will be searched in the backend/object storage only.
+      # Time ranges between query_backend_after and now will be queried from the metrics-generators.
+      query_backend_after: 30m
+      # The target length of time for each job to handle when querying the backend.
+      interval: 5m
+      # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
+      # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
+      # NOTE: `duration_slo` and `throughput_bytes_slo` both must be configured for it to work
+      duration_slo: 0s
+      # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
+      # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
+      throughput_bytes_slo: 0
   autoscaling:
     # -- Enable autoscaling for the query-frontend
     enabled: false
@@ -1359,8 +1376,13 @@ config: |
     trace_by_id:
       query_shards: {{ .Values.queryFrontend.config.trace_by_id.query_shards }}
     metrics:
+      concurrent_jobs:  {{ .Values.queryFrontend.config.metrics.concurrent_jobs }}
+      target_bytes_per_job:  {{ .Values.queryFrontend.config.metrics.target_bytes_per_job }}
       max_duration: {{ .Values.queryFrontend.config.metrics.max_duration }}
-
+      query_backend_after: {{ .Values.queryFrontend.config.metrics.query_backend_after }}
+      interval: {{ .Values.queryFrontend.config.metrics.interval }}
+      duration_slo: {{ .Values.queryFrontend.config.metrics.duration_slo }}
+      throughput_bytes_slo: {{ .Values.queryFrontend.config.metrics.throughput_bytes_slo }}
   ingester:
     lifecycler:
       ring: