From f26d65ab3b461228bf16496c00ea74a14a22b707 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 14 Aug 2024 12:13:19 +0200 Subject: [PATCH 01/14] Remove mimir dashboards and replace them with copied from tempo mixin --- .../tempo-alertmanager.json | 3519 -------- src/grafana_dashboards/tempo-compactor.json | 3031 ------- src/grafana_dashboards/tempo-config.json | 463 -- .../tempo-object-store.json | 1334 --- src/grafana_dashboards/tempo-operational.json | 7115 +++++++++++++++++ src/grafana_dashboards/tempo-overrides.json | 489 -- src/grafana_dashboards/tempo-overview.json | 2085 ----- src/grafana_dashboards/tempo-queries.json | 3986 --------- src/grafana_dashboards/tempo-reads.json | 1612 ++++ src/grafana_dashboards/tempo-resources.json | 2333 ++++++ .../tempo-rollout-progress.json | 1555 ++++ src/grafana_dashboards/tempo-ruler.json | 2555 ------ src/grafana_dashboards/tempo-tenants.json | 1149 +++ src/grafana_dashboards/tempo-writes.json | 1732 ++++ 14 files changed, 15496 insertions(+), 17462 deletions(-) delete mode 100644 src/grafana_dashboards/tempo-alertmanager.json delete mode 100644 src/grafana_dashboards/tempo-compactor.json delete mode 100644 src/grafana_dashboards/tempo-config.json delete mode 100644 src/grafana_dashboards/tempo-object-store.json create mode 100644 src/grafana_dashboards/tempo-operational.json delete mode 100644 src/grafana_dashboards/tempo-overrides.json delete mode 100644 src/grafana_dashboards/tempo-overview.json delete mode 100644 src/grafana_dashboards/tempo-queries.json create mode 100644 src/grafana_dashboards/tempo-reads.json create mode 100644 src/grafana_dashboards/tempo-resources.json create mode 100644 src/grafana_dashboards/tempo-rollout-progress.json delete mode 100644 src/grafana_dashboards/tempo-ruler.json create mode 100644 src/grafana_dashboards/tempo-tenants.json create mode 100644 src/grafana_dashboards/tempo-writes.json diff --git a/src/grafana_dashboards/tempo-alertmanager.json b/src/grafana_dashboards/tempo-alertmanager.json deleted file mode 100644 index 6847649..0000000 --- a/src/grafana_dashboards/tempo-alertmanager.json +++ /dev/null @@ -1,3519 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 31, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Headlines", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 1, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(cortex_alertmanager_alerts{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Total alerts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 2, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(cortex_alertmanager_silences{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Total silences", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 3, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "max(cortex_alertmanager_tenants_discovered{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Tenants", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 32, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Alertmanager Distributor", - "type": "row" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "1000 * sum(rate(cortex_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval])) / sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 33, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Alerts received", - "type": "row" - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_alerts_received_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_alerts_invalid_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_alerts_invalid_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "APS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 34, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Alerts grouping", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "cortex_alertmanager_dispatcher_aggregation_groups{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{juju_unit}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "per unit Active Aggregation Groups", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 35, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Alert notifications", - "type": "row" - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_notifications_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "NPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 29 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "(sum by(integration) (rate(cortex_alertmanager_notifications_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum by(integration) (rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) > 0 or on() vector(0)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "success - {{ integration }}", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(integration) (rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed - {{ integration }}", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "NPS by integration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 29 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_alertmanager_notification_latency_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_alertmanager_notification_latency_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1000 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 36, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Alertmanager Configuration Object Store (Alertmanager accesses)", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 37 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 37 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 37 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 37 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 37, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 45 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 45 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 45 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 45 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 38, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Replication", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 53 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "max by(juju_unit) (cortex_alertmanager_tenants_owned{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{juju_unit}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Per unit tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 53 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(juju_unit) (cortex_alertmanager_alerts{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{juju_unit}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Per unit alerts", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 53 - }, - "hiddenSeries": false, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(juju_unit) (cortex_alertmanager_silences{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{juju_unit}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Per unit silences", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 60 - }, - "id": 39, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Tenant configuration sync", - "type": "row" - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 61 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_sync_configs_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_sync_configs_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Syncs/sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 61 - }, - "hiddenSeries": false, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{reason}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Syncs/sec (by reason)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 61 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_ring_check_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "errors", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Ring check errors/sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 40, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Sharding initial state sync", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 69 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{outcome}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Initial syncs /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 69 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1000 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Initial sync duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 69 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Fetch state from other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 76 - }, - "id": 41, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Sharding runtime state sync", - "type": "row" - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 77 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_replication_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_state_replication_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_replication_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Replicate state to other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 77 - }, - "hiddenSeries": false, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_partial_state_merges_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_partial_state_merges_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_partial_state_merges_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Merge state from other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 77 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_persist_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_state_persist_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Persist state to remote storage /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ], - "value": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Alertmanager", - "uid": "b0d38d318bbddd80476246d4930f9e55", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-compactor.json b/src/grafana_dashboards/tempo-compactor.json deleted file mode 100644 index 32b3015..0000000 --- a/src/grafana_dashboards/tempo-compactor.json +++ /dev/null @@ -1,3031 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 23, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Summary", - "type": "row" - }, - { - "aliasColors": { - "completed": "#7EB26D", - "failed": "#E24D42", - "started": "#34CCEB" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n", - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_runs_started_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "started", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_runs_completed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "completed", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_runs_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Per-instance runs / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\nReset to 0 after the compaction run is completed for all tenants in the shard.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "(\n cortex_compactor_tenants_processing_succeeded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} +\n cortex_compactor_tenants_processing_failed{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} +\n cortex_compactor_tenants_skipped{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}\n)\n/\ncortex_compactor_tenants_discovered{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{juju_unit}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tenants compaction progress", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Longest time since last successful run\nDisplays the amount of time since the most recent successful execution\nof the compactor.\nThe value shown will be for the compactor replica that has the longest time since its\nlast successful run.\nThe table to the right shows a summary for all compactor replicas.\n\nIf there is no time value, one of the following messages might appear:\n\n- If you see \"No compactor data\" in this panel, that means that no compactors are active yet.\n\n- If you see \"No successful runs\" in this panel, that means that compactors are active, but none\n of them were successfully executed yet.\n\nThese might be expected - for example, if you just recently restarted your compactors,\nthey might not have had a chance to complete their first compaction run.\nHowever, if these messages persist, you should check the health of your compactors.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [], - "noValue": "No compactor data", - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Last run" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "from": "-Infinity", - "result": { - "color": "text", - "text": "No successful runs since startup yet" - }, - "to": 0 - }, - "type": "range" - } - ] - }, - { - "id": "color", - "value": { - "mode": "thresholds" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 7200 - }, - { - "color": "orange", - "value": 21600 - }, - { - "color": "red", - "value": 43200 - } - ] - } - } - ] - } - ] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "first" - ], - "fields": "/^Last run$/", - "values": false - }, - "textMode": "value" - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "max by(juju_unit)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h])\n)\n", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "title": "Longest time since last successful run", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transformations": [ - { - "id": "organize", - "options": { - "renameByName": { - "Value": "Last run", - "juju_unit": "Compactor" - } - } - }, - { - "id": "sortBy", - "options": { - "sort": [ - { - "desc": true, - "field": "Last run" - } - ] - } - } - ], - "type": "stat", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Last successful run per-compactor replica\nDisplays the compactor replicas, and for each, shows how long it has been since\nits last successful compaction run.\n\nThe value in the status column is based on how long it has been since the last successful compaction.\n\n- Okay: less than 2 hours\n- Delayed: more than 2 hours\n- Late: more than 6 hours\n- Very late: more than 12 hours\n\nIf the status of any compactor replicas are *Late* or *Very late*, check their health.\n\n", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": "-Infinity", - "result": { - "color": "transparent", - "text": "N/A" - }, - "to": 0 - }, - "type": "range" - }, - { - "options": { - "from": 0, - "result": { - "color": "green", - "text": "Ok" - }, - "to": 7200 - }, - "type": "range" - }, - { - "options": { - "from": 7200, - "result": { - "color": "yellow", - "text": "Delayed" - }, - "to": 21600 - }, - "type": "range" - }, - { - "options": { - "from": 21600, - "result": { - "color": "orange", - "text": "Late" - }, - "to": 43200 - }, - "type": "range" - }, - { - "options": { - "from": 43200, - "result": { - "color": "red", - "text": "Very late" - }, - "to": "Infinity" - }, - "type": "range" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "transparent", - "text": "Unknown" - } - }, - "type": "special" - } - ] - }, - { - "id": "custom.width", - "value": 86 - }, - { - "id": "custom.align", - "value": "center" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Last run" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "custom.width", - "value": 74 - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": "-Infinity", - "result": { - "text": "Never" - }, - "to": 0 - }, - "type": "range" - } - ] - } - ] - } - ] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "max by(juju_unit)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h])\n)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "Last run", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "title": "Last successful run per-compactor replica", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transformations": [ - { - "id": "organize", - "options": { - "renameByName": { - "Value": "Last run", - "juju_unit": "Compactor" - } - } - }, - { - "id": "sortBy", - "options": { - "sort": [ - { - "desc": true, - "field": "Last run" - } - ] - } - }, - { - "id": "calculateField", - "options": { - "alias": "One", - "binary": { - "left": "Last run", - "operator": "/", - "right": "Last run" - }, - "mode": "binary", - "replaceFields": false - } - }, - { - "id": "calculateField", - "options": { - "alias": "Status", - "binary": { - "left": "Last run", - "operator": "*", - "right": "One" - }, - "mode": "binary", - "replaceFields": false - } - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Compactor", - "Last run", - "Status" - ] - } - } - } - ], - "type": "table", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ] - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 24, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(prometheus_tsdb_compactions_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "compactions", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TSDB compactions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### TSDB compaction duration\nDisplay the amount of time that it has taken to run a single TSDB compaction.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TSDB compaction duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 25, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "avg(max by(user) (cortex_bucket_blocks_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Average blocks / tenant", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{user}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tenants with largest number of blocks", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 26, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Garbage collector", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "blocks", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks marked for deletion / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks deletions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 27, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Metadata sync", - "type": "row" - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_meta_syncs_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n-\nsum(rate(cortex_compactor_meta_sync_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Metadata syncs / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Metadata sync duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 28, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Object Store", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 41 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 41 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 41 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 41 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 29, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 49 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 49 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 49 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 49 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency of op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 30, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Key-value store for compactors ring", - "type": "row" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 57 - }, - "hiddenSeries": false, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 57 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ], - "value": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Compactor", - "uid": "1b3443aea86db629e6efdb7d05c53823", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-config.json b/src/grafana_dashboards/tempo-config.json deleted file mode 100644 index 381aa17..0000000 --- a/src/grafana_dashboards/tempo-config.json +++ /dev/null @@ -1,463 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 3, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Startup config file", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "count by(sha256) (cortex_config_hash{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "sha256:{{sha256}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Startup config file hashes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "instances", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 4, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Runtime config file", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "count by(sha256) (cortex_runtime_config_hash{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "sha256:{{sha256}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Runtime config file hashes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "instances", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus datasource", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Config", - "uid": "5d9d0b4724c0f80d68467088ec61e003", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-object-store.json b/src/grafana_dashboards/tempo-object-store.json deleted file mode 100644 index d8da615..0000000 --- a/src/grafana_dashboards/tempo-object-store.json +++ /dev/null @@ -1,1334 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Components", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{component}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "RPS / component", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{component}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Error rate / component", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 12, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Operations", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "RPS / operation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Error rate / operation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 13, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 17 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 17 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 14, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 25 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 25 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ], - "value": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Object Store", - "uid": "e1324ee2a434f4158c00a9ee279d3292", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json new file mode 100644 index 0000000..2c511ac --- /dev/null +++ b/src/grafana_dashboards/tempo-operational.json @@ -0,0 +1,7115 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "uid": "$logsds" + }, + "enable": true, + "expr": "{cluster=\"$cluster\", diff_namespace=\"$namespace\", container=\"kube-diff-logger\"}", + "hide": true, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "diffs", + "showIn": 0, + "target": { + + } + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 38, + "panels": [ + + ], + "title": "General", + "type": "row" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "gcs", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "go heap", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "goroutines", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "cpu", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "working set", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "rx-{{pod}}", + "refId": "A" + }, + { + "expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "tx-{{pod}}", + "refId": "B" + } + ], + "title": "tx/rx", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}", + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" + } + ], + "title": "data volume free", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{exported_pod}}", + "refId": "A" + } + ], + "title": "bad words", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 49, + "panels": [ + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 2 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "%age total work queue", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 2 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "legendFormat": "compaction_err", + "refId": "B" + }, + { + "expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "legendFormat": "retention_err", + "refId": "C" + }, + { + "expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "legendFormat": "blocklist_err", + "refId": "D" + } + ], + "title": "maintenance errors", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 2 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Blocklist Poll Duration", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 2 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}) by (tenant)", + "instant": false, + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Blocklist Length", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 2 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "retention duration", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 2 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "deleted", + "refId": "A" + }, + { + "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "marked_for_deletion", + "refId": "B" + } + ], + "title": "retention", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 2 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Container Restarts", + "type": "timeseries" + } + ], + "title": "Maintenance", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 21, + "panels": [ + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 8 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Queries/Sec (cortex-gw)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 8 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Gateway)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 8 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (cortex-gw)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 8 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "hide": false, + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "hide": false, + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Gateway)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 90, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Queries/Sec (Query Frontend)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 13 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Query Frontend)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 13 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (Query Frontend)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 13 + }, + "id": 94, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Query Frontend)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 18 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Queries/Sec (Querier)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 18 + }, + "id": 89, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Querier)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 18 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (Querier)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 18 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Querier)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 23 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Queries/Sec (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 23 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 23 + }, + "id": 100, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 23 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 28 + }, + "id": 102, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_metrics_summary\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Metrics Summary QPS (cortex-gw)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 28 + }, + "id": 103, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "Metrics Summary Latency (Gateway)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 28 + }, + "id": 110, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_metrics.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "range": true, + "refId": "A", + "stepMode": "min" + } + ], + "title": "Metrics Summary QPS (Query Frontend)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 28 + }, + "id": 111, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A", + "stepMode": "min" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B", + "stepMode": "min" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C", + "stepMode": "min" + } + ], + "title": "Metrics Summary Latency (Query Frontend)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 33 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_metrics.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}} {{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Metrics Summary QPS (Querier)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 33 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "Metrics Summary Latency (Querier)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 33 + }, + "id": 106, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.MetricsGenerator/GetMetrics\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "GetMetrics QPS (Metrics Generator)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 33 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "GetMetrics Latency (Metrics Generator)", + "type": "timeseries" + } + ], + "title": "Read", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P666011C0B63BDCA4" + }, + "editorMode": "code", + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Spans Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Traces Created", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 4 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1h]))", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Blocks Flushed (1h)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 4 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "increase(tempo_ingester_failed_flushes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Failed Flushes", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 4 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Blocks Cleared (1h)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 4 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Flush Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 4 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "sum(tempo_distributor_queue_length{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\".*/distributor\"}) by (name)", + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Distributor Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 10, + "w": 7, + "x": 0, + "y": 9 + }, + "id": 71, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)", + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Pushes/sec (gateway)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 7, + "y": 9 + }, + "id": 72, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "refId": "A" + }, + { + "expr": "sum(rate(tempo_receiver_refused_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "refused", + "refId": "B" + } + ], + "title": "OTEL Distributor Spans/second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 13, + "y": 9 + }, + "id": 79, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Push Latency (Gateway)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 13, + "y": 14 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Push Latency (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 7, + "y": 19 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (reason)", + "interval": "", + "legendFormat": "refused {{reason}}", + "range": true, + "refId": "B" + } + ], + "title": "Generator Spans/second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 13, + "y": 19 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "cortex-ops-01" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "Push Latency (Generator)", + "type": "timeseries" + } + ], + "title": "Write", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 74, + "panels": [ + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 75, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)", + "interval": "", + "legendFormat": "{{status_code}}-{{method}}", + "refId": "A" + } + ], + "title": "Requests/Second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "interval": "", + "legendFormat": ".99-{{method}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "interval": "", + "legendFormat": ".9-{{method}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "interval": "", + "legendFormat": ".5-{{method}}", + "refId": "C" + } + ], + "title": "Latency By Operation", + "type": "timeseries" + } + ], + "title": "Memcached", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 28, + "panels": [ + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, operation)", + "interval": "", + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "title": "Requests/Second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "title": "Latency By Operation", + "type": "timeseries" + } + ], + "title": "Backend", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 56, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 58, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P666011C0B63BDCA4" + }, + "expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Gossip Health", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 17 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "tempo_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Node Health", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 17 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "min(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "interval": "", + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "max(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Member Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 17 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P666011C0B63BDCA4" + }, + "expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "interval": "", + "legendFormat": "min", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P666011C0B63BDCA4" + }, + "expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "KV Store Count", + "type": "timeseries" + } + ], + "title": "Ring/Memberlist", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 69, + "panels": [ + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 77, + "options": { + "graph": { + + }, + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.0-14141pre", + "targets": [ + { + "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{error}}", + "refId": "A" + } + ], + "title": "Vulture Query Errors", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{secondsago}}", + "refId": "A" + } + ], + "title": "Average Vulture Query Errors", + "type": "bargauge" + } + ], + "title": "Vulture", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 81, + "panels": [ + { + "datasource": { + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 19 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Objects Combined / s", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 19 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Objects Written / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 19 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P666011C0B63BDCA4" + }, + "editorMode": "builder", + "expr": "sum(rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bytes Written / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 19 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P666011C0B63BDCA4" + }, + "editorMode": "code", + "expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[5m])) by (level)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Blocks Compacted", + "type": "timeseries" + } + ], + "title": "Compactor", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "ds", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "loki-ops", + "value": "loki-ops" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "logsds", + "options": [ + + ], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "ops-us-east-0", + "value": "ops-us-east-0" + }, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(tempo_build_info, cluster)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": { + "query": "label_values(tempo_build_info, cluster)", + "refId": "ops-cortex-cluster-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "tempo-ops", + "value": "tempo-ops" + }, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": { + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refId": "ops-cortex-namespace-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "multi": false, + "name": "component", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "compactor", + "value": "compactor" + }, + { + "selected": false, + "text": "distributor", + "value": "distributor" + }, + { + "selected": false, + "text": "ingester", + "value": "ingester" + }, + { + "selected": false, + "text": "metrics-generator", + "value": "metrics-generator" + }, + { + "selected": false, + "text": "query-frontend", + "value": "query-frontend" + }, + { + "selected": false, + "text": "querier", + "value": "querier" + }, + { + "selected": false, + "text": "cortex-gw", + "value": "cortex-gw" + }, + { + "selected": false, + "text": "cortex-gw-internal", + "value": "cortex-gw-internal" + } + ], + "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier,cortex-gw,cortex-gw-internal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Tempo Operational", + "uid": "a6175b9cc7ec20591890117c39580030", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-overrides.json b/src/grafana_dashboards/tempo-overrides.json deleted file mode 100644 index 47edc5e..0000000 --- a/src/grafana_dashboards/tempo-overrides.json +++ /dev/null @@ -1,489 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 3, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "max by(limit_name) (cortex_limits_defaults{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Defaults", - "transformations": [ - { - "id": "labelsToFields", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { - "Value": 1, - "limit_name": 0 - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "limit_name" - } - ] - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 4, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 2, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "max by(user, limit_name) (cortex_limits_overrides{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",user=~\"${tenant_id}\"})", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Per-tenant overrides", - "transformations": [ - { - "id": "labelsToFields", - "options": { - "mode": "columns", - "valueLabel": "limit_name" - } - }, - { - "id": "merge", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { - "user": 0 - } - } - } - ], - "type": "table" - } - ], - "refresh": "", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ], - "value": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": ".*", - "value": ".*" - }, - "hide": 0, - "label": "Tenant ID", - "name": "tenant_id", - "options": [ - { - "selected": true, - "text": ".*", - "value": ".*" - } - ], - "query": ".*", - "skipUrlSync": false, - "type": "textbox" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Overrides", - "uid": "1e2c358600ac53f09faea133f811b5bb", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-overview.json b/src/grafana_dashboards/tempo-overview.json deleted file mode 100644 index 7f38671..0000000 --- a/src/grafana_dashboards/tempo-overview.json +++ /dev/null @@ -1,2085 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 20, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Tempo cluster health", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The 'Status' panel shows an overview on the cluster health over the time.\nTo investigate failures, see a specific dashboard:\n\n- Writes\n- Reads\n- Rule evaluations\n- Alerting notifications\n- Object storage\n", - "mode": "markdown" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#7EB26D", - "value": null - }, - { - "color": "#EAB839", - "value": 0.01 - }, - { - "color": "#E24D42", - "value": 0.05 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 6, - "y": 1 - }, - "id": 2, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "exemplar": false, - "expr": "(sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((distributor|cortex|tempo|tempo-write.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.*|error\"}[$__rate_interval])) or vector(0)) / sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((distributor|cortex|tempo|tempo-write.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "Writes", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "exemplar": false, - "expr": "(sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((query-frontend.*|cortex|tempo|tempo-read.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.*\"}[$__rate_interval])) or vector(0)) / sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((query-frontend.*|cortex|tempo|tempo-read.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "Reads", - "range": true, - "refId": "B" - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "exemplar": false, - "expr": "((sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) + sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) or vector(0)) / sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "Rule evaluations", - "range": true, - "refId": "C" - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "exemplar": false, - "expr": "(((sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) or vector(0)) + ((sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\",job=~\"($namespace)/((alertmanager|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})) or vector(0))) / (((sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) or vector(0)) + ((sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\",job=~\"($namespace)/((alertmanager|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})) or vector(0)))", - "instant": false, - "legendFormat": "Alerting notifications", - "range": true, - "refId": "D" - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "exemplar": false, - "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "Object storage", - "range": true, - "refId": "E" - } - ], - "title": "Status", - "type": "state-timeline" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 3, - "options": { - "alertInstanceLabelFilter": "juju_application=\"$juju_application\", juju_model=\"$juju_model\",juju_model_uuid=\"$juju_model_uuid\",juju_unit=\"$juju_unit\"", - "alertName": "Tempo", - "dashboardAlerts": false, - "groupBy": [], - "groupMode": "default", - "maxItems": 100, - "sortOrder": 3, - "stateFilter": { - "error": true, - "firing": true, - "noData": false, - "normal": false, - "pending": false - }, - "viewMode": "list" - }, - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Firing alerts", - "type": "alertlist" - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 21, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Writes", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 9 - }, - "id": 4, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "These panels show an overview on the write path. \nTo examine the write path in detail, see a specific dashboard:\n\n- Writes\n- Writes resources\n- Writes networking\n- Overview resources\n- Overview networking\n", - "mode": "markdown" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 9 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "1000 * sum(rate(cortex_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval])) / sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 9 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_distributor_received_samples{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "samples / sec", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_distributor_received_exemplars{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "exemplars / sec", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Ingestion / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "cps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 22, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Reads", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 17 - }, - "id": 8, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "These panels show an overview on the read path. \nTo examine the read path in detail, see a specific dashboard:\n\n- Reads\n- Reads resources\n- Reads networking\n- Overview resources\n- Overview networking\n- Queries\n- Compactor\n", - "mode": "markdown" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 17 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "1000 * sum(rate(cortex_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])) / sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 17 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "instant queries", - "color": "#429D48" - }, - { - "alias": "range queries", - "color": "#F1C731" - }, - { - "alias": "\"label names\" queries", - "color": "#2A66CF" - }, - { - "alias": "\"label values\" queries", - "color": "#9E44C1" - }, - { - "alias": "series queries", - "color": "#FFAB57" - }, - { - "alias": "remote read queries", - "color": "#C79424" - }, - { - "alias": "metadata queries", - "color": "#84D586" - }, - { - "alias": "exemplar queries", - "color": "#A1C4FC" - }, - { - "alias": "other", - "color": "#C788DE" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_query\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "instant queries", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "range queries", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "\"label names\" queries", - "range": true, - "refId": "C", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "\"label values\" queries", - "range": true, - "refId": "D", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "series queries", - "range": true, - "refId": "E", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_read\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "remote read queries", - "range": true, - "refId": "F", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_metadata\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "metadata queries", - "range": true, - "refId": "G", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_query_exemplars\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "exemplar queries", - "range": true, - "refId": "H", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route!~\".*(query|query_range|label.*|series|read|metadata|query_exemplars)\",route=~\"(prometheus|api_prom)_api_v1_.*\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "other", - "range": true, - "refId": "I", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queries / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 23, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Recording and alerting rules", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 25 - }, - "id": 12, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "These panels show an overview on the recording and alerting rules evaluation.\nTo examine the rules evaluation and alerts notifications in detail, see a specific dashboard:\n\n- Ruler\n- Alertmanager\n- Alertmanager resources\n- Overview resources\n- Overview networking\n", - "mode": "markdown" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "aliasColors": { - "failed": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 25 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "success", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "missed", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Rule evaluations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 25 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Rule evaluations latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 25 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_notifications_sent_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_prometheus_notifications_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_notifications_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Alerting notifications sent to Alertmanager / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 24, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Long-term storage (object storage)", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 33 - }, - "id": 16, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "These panels show an overview on the long-term storage (object storage).\nTo examine the storage in detail, see a specific dashboard:\n\n- Object store\n- Compactor\n", - "mode": "markdown" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 33 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 33 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "attributes", - "color": "#429D48" - }, - { - "alias": "delete", - "color": "#F1C731" - }, - { - "alias": "exists", - "color": "#2A66CF" - }, - { - "alias": "get", - "color": "#9E44C1" - }, - { - "alias": "get_range", - "color": "#FFAB57" - }, - { - "alias": "iter", - "color": "#C79424" - }, - { - "alias": "upload", - "color": "#84D586" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 33 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$prometheusds" - }, - "editorMode": "code", - "expr": "sum(max by(user) (max_over_time(cortex_bucket_blocks_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[15m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "blocks", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total number of blocks in the storage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ], - "value": [ - "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Overview", - "uid": "ffcd83628d7d4b5a03d1cafd159e6c9c", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-queries.json b/src/grafana_dashboards/tempo-queries.json deleted file mode 100644 index 16bad0c..0000000 --- a/src/grafana_dashboards/tempo-queries.json +++ /dev/null @@ -1,3986 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 37, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Query-frontend", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queue duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_query_frontend_retries_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Retries", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 1 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (cortex_query_frontend_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{user}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queue length (per user)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 38, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Query-scheduler", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queue duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (cortex_query_scheduler_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{user}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queue length (per user)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 39, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Query-frontend - query splitting and results cache", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Intervals per query\nThe average number of split queries (partitioned by time) executed a single input query.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_frontend_split_queries_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", method=\"split_by_interval_and_results_cache\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "splitting rate", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Intervals per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 17 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_hits{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n/\nsum (\n rate(cortex_cache_fetched_keys{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Hit ratio", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Query results cache hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_fetched_keys{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n-\nsum (\n rate(cortex_cache_hits{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Missed query results per second", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Query results cache misses", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Query results cache skipped\nThe % of queries whose results could not be cached.\nIt is tracked for each split query when the splitting by interval is enabled.\n\n", - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 17 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_frontend_query_result_cache_skipped_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (reason) /\nignoring (reason) group_left sum(rate(cortex_frontend_query_result_cache_attempted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{reason}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Query results cache skipped", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 40, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Query-frontend - query sharding", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Sharded queries ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel only takes into account the type of queries that are supported by query sharding (eg. range queries).\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "sharded queries ratio", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Sharded queries ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Number of sharded queries per query\nThe number of sharded queries that have been executed for a single input query. It only tracks queries that\nhave been successfully rewritten in a shardable way.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|tempo|tempo-read.*))\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Number of sharded queries per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 41, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Ingester", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 33 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(cortex_ingester_queried_series_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_ingester_queried_series_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "1 * sum(rate(cortex_ingester_queried_series_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_ingester_queried_series_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Series per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 33 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(cortex_ingester_queried_samples_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_ingester_queried_samples_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "1 * sum(rate(cortex_ingester_queried_samples_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_ingester_queried_samples_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Samples per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 33 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(cortex_ingester_queried_exemplars_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_ingester_queried_exemplars_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "1 * sum(rate(cortex_ingester_queried_exemplars_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_ingester_queried_exemplars_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Exemplars per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 42, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Querier", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 41 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Number of store-gateways hit per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 41 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Refetches of missing blocks per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "Failure Rate": "#E24D42" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 41 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Failure Rate", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Consistency checks failed", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 43, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 49 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "max(cortex_bucket_index_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Max", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "min(cortex_bucket_index_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Min", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "avg(cortex_bucket_index_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket indexes loaded (per querier)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 49 - }, - "hiddenSeries": false, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_index_loads_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_index_load_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket indexes load / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 49 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket indexes load latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 44, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Store-gateway", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 57 - }, - "hiddenSeries": false, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "blocks", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks queried / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 57 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(data_type) (\n # Exclude \"chunks refetched\".\n rate(cortex_bucket_store_series_data_size_fetched_bytes_sum{component=\"store-gateway\", stage!=\"refetched\", juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{data_type}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Data fetched / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "binBps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 57 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(data_type) (\n # Exclude \"chunks processed\" to only count \"chunks returned\", other than postings and series.\n rate(cortex_bucket_store_series_data_size_touched_bytes_sum{component=\"store-gateway\", stage!=\"processed\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{data_type}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Data touched / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "binBps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 45, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 65 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{stage}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Series request average latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 65 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(cortex_bucket_store_series_request_stage_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{stage}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Series request 99th percentile latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Series batch preloading efficiency\nThis panel shows the % of time reduced by preloading, for Series() requests which have been\nsplit to 2+ batches. If a Series() request is served within a single batch, then preloading\nis not triggered, and thus not counted in this measurement.\n\n", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 65 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "# Clamping min to 0 because if preloading not useful at all, then the actual value we get is\n# slightly negative because of the small overhead introduced by preloading.\nclamp_min(1 - (\n sum(rate(cortex_bucket_store_series_batch_preloading_wait_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n /\n sum(rate(cortex_bucket_store_series_batch_preloading_load_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n), 0)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "% of time reduced by preloading", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Series batch preloading efficiency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 72 - }, - "id": 46, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 73 - }, - "hiddenSeries": false, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "cortex_bucket_store_blocks_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks currently loaded", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 73 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks loaded / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "failed": "#E24D42", - "successful": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 73 - }, - "hiddenSeries": false, - "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "successful", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks dropped / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 80 - }, - "id": 47, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 81 - }, - "hiddenSeries": false, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "cortex_bucket_store_indexheader_lazy_load_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} - cortex_bucket_store_indexheader_lazy_unload_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Lazy loaded index-headers", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 81 - }, - "hiddenSeries": false, - "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index-header lazy load duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 88 - }, - "id": 48, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 89 - }, - "hiddenSeries": false, - "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "hit ratio", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Series hash cache hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 89 - }, - "hiddenSeries": false, - "id": 35, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(thanos_store_index_cache_hits_total{item_type=\"ExpandedPostings\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum(rate(thanos_store_index_cache_requests_total{item_type=\"ExpandedPostings\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "hit ratio", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "ExpandedPostings cache hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 89 - }, - "hiddenSeries": false, - "id": 36, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_cache_memory_hits_total{name=\"chunks-attributes-cache\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum(rate(cortex_cache_memory_requests_total{name=\"chunks-attributes-cache\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "hit ratio", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunks attributes in-memory cache hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "hide": 0, - "includeAll": true, - "label": "Loki datasource", - "multi": true, - "name": "lokids", - "options": [], - "query": "loki", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus datasource", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Queries", - "uid": "b3abe8d5c040395cc36615cb4334c92d", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json new file mode 100644 index 0000000..2564b7e --- /dev/null +++ b/src/grafana_dashboards/tempo-reads.json @@ -0,0 +1,1612 @@ +{ + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Gateway", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query Frontend", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (le,endpoint)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (le,endpoint)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (endpoint)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier External Endpoint", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/tempopb.Querier/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memcached", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Backend", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ + + ], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [ + + ], + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Reads", + "uid": "", + "version": 0 +} \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-resources.json b/src/grafana_dashboards/tempo-resources.json new file mode 100644 index 0000000..57a6ff7 --- /dev/null +++ b/src/grafana_dashboards/tempo-resources.json @@ -0,0 +1,2333 @@ +{ + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Gateway", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distributor", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Metrics-generator", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query Frontend", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "legendFormat": "request", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Compactor", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ + + ], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [ + + ], + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Resources", + "uid": "", + "version": 0 +} \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-rollout-progress.json b/src/grafana_dashboards/tempo-rollout-progress.json new file mode 100644 index 0000000..3481773 --- /dev/null +++ b/src/grafana_dashboards/tempo-rollout-progress.json @@ -0,0 +1,1555 @@ +{ + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "yellow", + "value": 0.999 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "percentunit" + } + }, + "fill": 1, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", + "format": null, + "intervalFactor": null, + "legendFormat": "{{tempo_service}}", + "legendLink": null, + "step": null + }, + { + "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", + "format": null, + "intervalFactor": null, + "legendFormat": "{{tempo_service}}", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rollout progress", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "bargauge", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 2, + "x": 10, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Writes - 2xx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Writes - 4xx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 0 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Writes - 5xx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Writes 99th latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 2, + "x": 10, + "y": 4 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Reads - 2xx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 4 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Reads - 4xx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 4 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Reads - 5xx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "noValue": "", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2.5 + } + ] + }, + "unit": "s" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))\n", + "format": null, + "instant": false, + "interval": "", + "intervalFactor": null, + "legendFormat": "", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Reads 99th latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "noValue": "All healthy", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [ + + ] + }, + "fill": 1, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 8 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "options": { + "text": { + "titleSize": 14, + "valueSize": 14 + } + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n", + "format": null, + "instant": true, + "interval": "", + "intervalFactor": null, + "legendFormat": "{{deployment}}", + "legendLink": null, + "step": null + }, + { + "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n", + "format": null, + "instant": true, + "interval": "", + "intervalFactor": null, + "legendFormat": "{{statefulset}}", + "legendLink": null, + "step": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Unhealthy pods", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "stat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "r.*" + }, + "properties": [ + { + "id": "custom.align", + "value": "center" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 10, + "y": 8 + }, + "id": 11, + "targets": [ + { + "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"version\", \"$1\", \"image\", \".*:(.+)-.*\"\n )\n)\n", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Pods count per version", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "version" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "container" + } + ] + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))[1h:])\n)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "writes", + "legendLink": null + }, + { + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))[1h:])\n)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "reads", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency vs 24h ago", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "refresh": "", + "rows": null, + "schemaVersion": 27, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ + + ], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [ + + ], + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Rollout progress", + "uid": "217b16a8c5966b32c770225dea289b19", + "version": 0 +} \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-ruler.json b/src/grafana_dashboards/tempo-ruler.json deleted file mode 100644 index 6cf76f4..0000000 --- a/src/grafana_dashboards/tempo-ruler.json +++ /dev/null @@ -1,2555 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 31, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Headlines", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 1, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(cortex_ruler_managers_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Active configurations", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 2, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(cortex_prometheus_rule_group_rules{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Total rules", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "description": "### Read from ingesters - QPS\nNote: Even while operating in Remote ruler mode you will still see values for this panel.\n\nThis is because the metrics are inclusive of intermediate services and are showing the requests that ultimately reach the ingesters.\n\nFor a more detailed view of the read path when using remote ruler mode, see the Remote ruler reads dashboard.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 3, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Read from ingesters - QPS", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 4, - "links": [], - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.1", - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Write to ingesters - QPS", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 32, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Rule evaluations global", - "type": "row" - }, - { - "aliasColors": { - "failed": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "success", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "missed", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Evaluations per second", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 33, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Writes (ingesters)", - "type": "row" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 13 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1000 / sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 34, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Reads (ingesters)", - "type": "row" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1000 / sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 35, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Ruler - key-value store for rulers ring", - "type": "row" - }, - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 10, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]))) * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval])) * 1000 / sum(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 36, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Ruler - blocks storage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 37 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Number of store-gateways hit per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 37 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "range": true, - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Refetches of missing blocks per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "Failures / sec": "#E24D42" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 37 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Failures / sec", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Consistency checks failed", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 37, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Notifications", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 45 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(user) (rate(cortex_prometheus_notifications_sent_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ user }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Delivery errors", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 45 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(user) (rate(cortex_prometheus_notifications_queue_capacity{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ user }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queue length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 45 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (increase(cortex_prometheus_notifications_dropped_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ user }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Dropped", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": true, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 38, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 53 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ user }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Missed iterations", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 53 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ user }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 53 - }, - "hiddenSeries": false, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ rule_group }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Failures", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Group evaluations", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 53 - }, - "id": 39, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 54 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ user }}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Rule evaluation per user", - "type": "row" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "hide": 0, - "includeAll": true, - "label": "Loki datasource", - "multi": true, - "name": "lokids", - "options": [], - "query": "loki", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "hide": 0, - "includeAll": true, - "label": "Prometheus datasource", - "multi": true, - "name": "prometheusds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "hide": 0, - "includeAll": true, - "label": "Juju unit", - "multi": true, - "name": "juju_unit", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "hide": 0, - "includeAll": true, - "label": "Juju application", - "multi": true, - "name": "juju_application", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "hide": 0, - "includeAll": true, - "label": "Juju model uuid", - "multi": true, - "name": "juju_model_uuid", - "options": [], - "query": { - "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${prometheusds}" - }, - "definition": "label_values(up,juju_model)", - "hide": 0, - "includeAll": true, - "label": "Juju model", - "multi": true, - "name": "juju_model", - "options": [], - "query": { - "query": "label_values(up,juju_model)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "2024-02-16T08:22:27.546Z", - "to": "2024-02-16T10:22:27.546Z" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Ruler", - "uid": "631e15d5d85afb2ca8e35d62984eeaa0", - "version": 1, - "weekStart": "" -} diff --git a/src/grafana_dashboards/tempo-tenants.json b/src/grafana_dashboards/tempo-tenants.json new file mode 100644 index 0000000..87387ed --- /dev/null +++ b/src/grafana_dashboards/tempo-tenants.json @@ -0,0 +1,1149 @@ +{ + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n) by (limit_name)\n", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Limits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Tenant info", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "dashes": true, + "fill": 0 + }, + { + "alias": "burst limit", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_distributor_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "received", + "legendLink": null + }, + { + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_rate_limit_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_rate_limit_bytes\"})\n) by (ingestion_rate_limit_bytes)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_burst_size_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_burst_size_bytes\"})\n) by (ingestion_burst_size_bytes)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "burst limit", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Distributor bytes/s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_distributor_spans_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "accepted", + "legendLink": null + }, + { + "expr": "sum(rate(tempo_discarded_spans_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval])) by (reason)", + "format": "time_series", + "interval": "1m", + "legendFormat": "refused {{ reason }}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Distributor spans/s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "global limit", + "dashes": true, + "fill": 0 + }, + { + "alias": "local limit", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(tempo_ingester_live_traces{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",tenant=\"$tenant\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "live traces", + "legendLink": null + }, + { + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_global_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_global_traces_per_user\"})\n) by (max_global_traces_per_user)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "global limit", + "legendLink": null + }, + { + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_local_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_local_traces_per_user\"})\n) by (max_local_traces_per_user)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "local limit", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Live traces", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingestion", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"traces\"}[$__rate_interval])) by (status)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{ status }}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Queries/s (ID lookup)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"search\"}[$__rate_interval])) by (status)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{ status }}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Queries/s (search)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Reads", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "show": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "length", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Blockslist length", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "show": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(tempodb_compaction_outstanding_blocks{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})\n/\ncount(tempo_build_info{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "blocks", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Outstanding compactions", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "show": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "rate", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Bytes/s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(tempo_metrics_generator_registry_active_series{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{ tenant }}", + "legendLink": null + }, + { + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"metrics_generator_max_active_series\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"metrics_generator_max_active_series\"})\n) by (metrics_generator_max_active_series)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Active series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Metrics generator", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ + + ], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [ + + ], + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "tenant", + "multi": false, + "name": "tenant", + "options": [ + + ], + "query": "label_values(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"}, tenant)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Tenants", + "uid": "", + "version": 0 +} \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json new file mode 100644 index 0000000..5d9b29b --- /dev/null +++ b/src/grafana_dashboards/tempo-writes.json @@ -0,0 +1,1732 @@ +{ + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Gateway", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", __name__=~\"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+\"},\n \"grpc_status\", \"$1\", \"__name__\", \"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)\"\n )\n [$__interval:$__interval]\n )\n)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{grpc_status}}", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": { + + } + }, + "overrides": [ + + ] + }, + "id": 4, + "options": { + "content": "Visit [Status codes and their use in gRPC](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md)\n\nCode | Number | Description\n---|---|---\nOK | 0 | Not an error; returned on success.\nCANCELLED | 1 | The operation was cancelled, typically by the caller.\nUNKNOWN | 2 | Unknown error. For example, this error may be returned when a Status value received from another address space belongs to an error space that is not known in this address space. Also errors raised by APIs that do not return enough error information may be converted to this error.\nINVALID_ARGUMENT | 3 | The client specified an invalid argument. Note that this differs from FAILED_PRECONDITION. INVALID_ARGUMENT indicates arguments that are problematic regardless of the state of the system (e.g., a malformed file name).\nDEADLINE_EXCEEDED | 4 | The deadline expired before the operation could complete. For operations that change the state of the system, this error may be returned even if the operation has completed successfully. For example, a successful response from a server could have been delayed long\nNOT_FOUND | 5 | Some requested entity (e.g., file or directory) was not found. Note to server developers: if a request is denied for an entire class of users, such as gradual feature rollout or undocumented allowlist, NOT_FOUND may be used. If a request is denied for some users within a class of users, such as user-based access control, PERMISSION_DENIED must be used.\nALREADY_EXISTS | 6 | The entity that a client attempted to create (e.g., file or directory) already exists.\nPERMISSION_DENIED | 7 | The caller does not have permission to execute the specified operation. PERMISSION_DENIED must not be used for rejections caused by exhausting some resource (use RESOURCE_EXHAUSTED instead for those errors). PERMISSION_DENIED must not be used if the caller can not be identified (use UNAUTHENTICATED instead for those errors). This error code does not imply the request is valid or the requested entity exists or satisfies other pre-conditions.\nRESOURCE_EXHAUSTED | 8 | Some resource has been exhausted, perhaps a per-user quota, or perhaps the entire file system is out of space.\nFAILED_PRECONDITION | 9 | The operation was rejected because the system is not in a state required for the operation's execution. For example, the directory to be deleted is non-empty, an rmdir operation is applied to a non-directory, etc. Service implementors can use the following guidelines to decide between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE: (a) Use UNAVAILABLE if the client can retry just the failing call. (b) Use ABORTED if the client should retry at a higher level (e.g., when a client-specified test-and-set fails, indicating the client should restart a read-modify-write sequence). (c) Use FAILED_PRECONDITION if the client should not retry until the system state has been explicitly fixed. E.g., if an \"rmdir\" fails because the directory is non-empty, FAILED_PRECONDITION should be returned since the client should not retry unless the files are deleted from the directory.\nABORTED | 10 | The operation was aborted, typically due to a concurrency issue such as a sequencer check failure or transaction abort. See the guidelines above for deciding between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE.\nOUT_OF_RANGE | 11 | The operation was attempted past the valid range. E.g., seeking or reading past end-of-file. Unlike INVALID_ARGUMENT, this error indicates a problem that may be fixed if the system state changes. For example, a 32-bit file system will generate INVALID_ARGUMENT if asked to read at an offset that is not in the range [0,2^32-1], but it will generate OUT_OF_RANGE if asked to read from an offset past the current file size. There is a fair bit of overlap between FAILED_PRECONDITION and OUT_OF_RANGE. We recommend using OUT_OF_RANGE (the more specific error) when it applies so that callers who are iterating through a space can easily look for an OUT_OF_RANGE error to detect when they are done.\nUNIMPLEMENTED | 12 | The operation is not implemented or is not supported/enabled in this service.\nINTERNAL | 13 | Internal errors. This means that some invariants expected by the underlying system have been broken. This error code is reserved for serious errors.\nUNAVAILABLE | 14 | The service is currently unavailable. This is most likely a transient condition, which can be corrected by retrying with a backoff. Note that it is not always safe to retry non-idempotent operations.\nDATA_LOSS | 15 | Unrecoverable data loss or corruption.\nUNAUTHENTICATED | 16 | The request does not have valid authentication credentials for the operation.\n", + "mode": "markdown" + }, + "span": 6, + "timeFrom": null, + "timeShift": null, + "title": "gRPC status codes", + "transparent": false, + "type": "text" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Envoy Proxy", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "accepted", + "legendLink": null + }, + { + "expr": "sum(rate(tempo_receiver_refused_spans{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "refused", + "legendLink": null + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Spans/Second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distributor", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memcached - Ingester", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Backend - Ingester", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memcached - Compactor", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Backend - Compactor", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ + + ], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [ + + ], + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Writes", + "uid": "", + "version": 0 +} \ No newline at end of file From 645ef27212a157167cdba96faa8570ce2bf136eb Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 14 Aug 2024 13:20:12 +0200 Subject: [PATCH 02/14] Replace datasource placeholders with prometheusds --- src/grafana_dashboards/tempo-operational.json | 136 +++++++++--------- src/grafana_dashboards/tempo-reads.json | 32 ++--- src/grafana_dashboards/tempo-resources.json | 46 +++--- .../tempo-rollout-progress.json | 28 ++-- src/grafana_dashboards/tempo-tenants.json | 26 ++-- src/grafana_dashboards/tempo-writes.json | 34 ++--- 6 files changed, 151 insertions(+), 151 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index 2c511ac..43c1797 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -77,7 +77,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -172,7 +172,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -267,7 +267,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -361,7 +361,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -453,7 +453,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -548,7 +548,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -651,7 +651,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -745,7 +745,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -854,7 +854,7 @@ "panels": [ { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -948,7 +948,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1052,7 +1052,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1158,7 +1158,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1255,7 +1255,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1362,7 +1362,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1463,7 +1463,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1577,7 +1577,7 @@ "panels": [ { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1672,7 +1672,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1778,7 +1778,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1875,7 +1875,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1989,7 +1989,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2084,7 +2084,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2190,7 +2190,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2287,7 +2287,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2399,7 +2399,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2494,7 +2494,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2600,7 +2600,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2697,7 +2697,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2809,7 +2809,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -2904,7 +2904,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3010,7 +3010,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3107,7 +3107,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3220,7 +3220,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3322,7 +3322,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3447,7 +3447,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3551,7 +3551,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3682,7 +3682,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3784,7 +3784,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -3909,7 +3909,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4011,7 +4011,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4154,7 +4154,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4256,7 +4256,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4357,7 +4357,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4452,7 +4452,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4547,7 +4547,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4642,7 +4642,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4750,7 +4750,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4851,7 +4851,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -4946,7 +4946,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5047,7 +5047,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5154,7 +5154,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5262,7 +5262,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5376,7 +5376,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5519,7 +5519,7 @@ "panels": [ { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5613,7 +5613,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5737,7 +5737,7 @@ "panels": [ { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5831,7 +5831,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -5953,7 +5953,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6051,7 +6051,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6145,7 +6145,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6246,7 +6246,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6372,7 +6372,7 @@ "panels": [ { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6460,7 +6460,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6542,7 +6542,7 @@ "panels": [ { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6633,7 +6633,7 @@ }, { "datasource": { - "uid": "$ds" + "uid": "${prometheusds}" }, "description": "", "fieldConfig": { @@ -6726,7 +6726,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6820,7 +6820,7 @@ { "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -6971,7 +6971,7 @@ }, "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "definition": "label_values(tempo_build_info, cluster)", "hide": 0, @@ -7002,7 +7002,7 @@ }, "datasource": { "type": "prometheus", - "uid": "$ds" + "uid": "${prometheusds}" }, "definition": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", "hide": 0, diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json index 2564b7e..58630b2 100644 --- a/src/grafana_dashboards/tempo-reads.json +++ b/src/grafana_dashboards/tempo-reads.json @@ -43,7 +43,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 1, "legend": { @@ -128,7 +128,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 2, "legend": { @@ -253,7 +253,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 3, "legend": { @@ -338,7 +338,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 4, "legend": { @@ -463,7 +463,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 5, "legend": { @@ -548,7 +548,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 6, "legend": { @@ -673,7 +673,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 7, "legend": { @@ -758,7 +758,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 8, "legend": { @@ -883,7 +883,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 9, "legend": { @@ -968,7 +968,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 10, "legend": { @@ -1093,7 +1093,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 11, "legend": { @@ -1178,7 +1178,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 12, "legend": { @@ -1303,7 +1303,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 13, "legend": { @@ -1388,7 +1388,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 14, "legend": { @@ -1525,7 +1525,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "cluster", @@ -1553,7 +1553,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "namespace", diff --git a/src/grafana_dashboards/tempo-resources.json b/src/grafana_dashboards/tempo-resources.json index 57a6ff7..44e353a 100644 --- a/src/grafana_dashboards/tempo-resources.json +++ b/src/grafana_dashboards/tempo-resources.json @@ -35,7 +35,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 1, "legend": { @@ -143,7 +143,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 2, "legend": { @@ -251,7 +251,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 3, "legend": { @@ -348,7 +348,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 4, "legend": { @@ -456,7 +456,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 5, "legend": { @@ -564,7 +564,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 6, "legend": { @@ -661,7 +661,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 7, "legend": { @@ -769,7 +769,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 8, "legend": { @@ -877,7 +877,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 9, "legend": { @@ -974,7 +974,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 10, "legend": { @@ -1082,7 +1082,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 11, "legend": { @@ -1190,7 +1190,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 12, "legend": { @@ -1287,7 +1287,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 13, "legend": { @@ -1395,7 +1395,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 14, "legend": { @@ -1503,7 +1503,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 15, "legend": { @@ -1600,7 +1600,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 16, "legend": { @@ -1708,7 +1708,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 17, "legend": { @@ -1816,7 +1816,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 18, "legend": { @@ -1913,7 +1913,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 19, "legend": { @@ -2021,7 +2021,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 20, "legend": { @@ -2129,7 +2129,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 21, "legend": { @@ -2246,7 +2246,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "cluster", @@ -2274,7 +2274,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "namespace", diff --git a/src/grafana_dashboards/tempo-rollout-progress.json b/src/grafana_dashboards/tempo-rollout-progress.json index 3481773..37a1313 100644 --- a/src/grafana_dashboards/tempo-rollout-progress.json +++ b/src/grafana_dashboards/tempo-rollout-progress.json @@ -30,7 +30,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -171,7 +171,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -287,7 +287,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -411,7 +411,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -531,7 +531,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -655,7 +655,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -771,7 +771,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -895,7 +895,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -1015,7 +1015,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -1139,7 +1139,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "defaults": { "color": { @@ -1273,7 +1273,7 @@ ] }, { - "datasource": "$datasource", + "datasource": "${prometheusds}", "fieldConfig": { "overrides": [ { @@ -1344,7 +1344,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "gridPos": { "h": 8, @@ -1468,7 +1468,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": false, "label": "cluster", @@ -1496,7 +1496,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": false, "label": "namespace", diff --git a/src/grafana_dashboards/tempo-tenants.json b/src/grafana_dashboards/tempo-tenants.json index 87387ed..384bc31 100644 --- a/src/grafana_dashboards/tempo-tenants.json +++ b/src/grafana_dashboards/tempo-tenants.json @@ -35,7 +35,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 1, "legend": { @@ -156,7 +156,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 2, "legend": { @@ -264,7 +264,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 3, "legend": { @@ -356,7 +356,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 4, "legend": { @@ -476,7 +476,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 5, "legend": { @@ -561,7 +561,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 6, "legend": { @@ -658,7 +658,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 7, "legend": { @@ -737,7 +737,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 8, "legend": { @@ -828,7 +828,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 9, "legend": { @@ -907,7 +907,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 10, "legend": { @@ -1035,7 +1035,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "cluster", @@ -1063,7 +1063,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "namespace", @@ -1090,7 +1090,7 @@ "text": "prod", "value": "prod" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": false, "label": "tenant", diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json index 5d9b29b..d7c128b 100644 --- a/src/grafana_dashboards/tempo-writes.json +++ b/src/grafana_dashboards/tempo-writes.json @@ -43,7 +43,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 1, "legend": { @@ -128,7 +128,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 2, "legend": { @@ -245,7 +245,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 3, "legend": { @@ -366,7 +366,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 5, "legend": { @@ -458,7 +458,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 6, "legend": { @@ -583,7 +583,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 7, "legend": { @@ -668,7 +668,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 8, "legend": { @@ -793,7 +793,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 9, "legend": { @@ -878,7 +878,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 10, "legend": { @@ -1003,7 +1003,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 11, "legend": { @@ -1088,7 +1088,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 12, "legend": { @@ -1213,7 +1213,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 13, "legend": { @@ -1298,7 +1298,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 14, "legend": { @@ -1423,7 +1423,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 10, "id": 15, "legend": { @@ -1508,7 +1508,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", + "datasource": "${prometheusds}", "fill": 1, "id": 16, "legend": { @@ -1645,7 +1645,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "cluster", @@ -1673,7 +1673,7 @@ "text": "All", "value": "$__all" }, - "datasource": "$datasource", + "datasource": "${prometheusds}", "hide": 0, "includeAll": true, "label": "namespace", From 113f9e39c40dceff727c9fc86d71d8022bee4a75 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 14 Aug 2024 14:07:33 +0200 Subject: [PATCH 03/14] Drop cluster parameter and adjust escaping characters that didn't propagate well --- src/grafana_dashboards/tempo-operational.json | 230 +++++++++--------- src/grafana_dashboards/tempo-reads.json | 56 ++--- src/grafana_dashboards/tempo-resources.json | 98 ++++---- .../tempo-rollout-progress.json | 30 +-- src/grafana_dashboards/tempo-tenants.json | 34 +-- src/grafana_dashboards/tempo-writes.json | 62 ++--- 6 files changed, 255 insertions(+), 255 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index 43c1797..60b6923 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -26,7 +26,7 @@ "uid": "$logsds" }, "enable": true, - "expr": "{cluster=\"$cluster\", diff_namespace=\"$namespace\", container=\"kube-diff-logger\"}", + "expr": "{diff_namespace='$namespace', container='kube-diff-logger'}", "hide": true, "iconColor": "rgba(255, 96, 96, 1)", "name": "diffs", @@ -161,7 +161,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])", + "expr": "rate(go_gc_duration_seconds_count{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -256,7 +256,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "go_memstats_heap_inuse_bytes{namespace='$namespace', job=~'$namespace/$component'}", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -351,7 +351,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "go_goroutines{namespace='$namespace', job=~'$namespace/$component'}", "legendFormat": "{{pod}}", "refId": "A" } @@ -441,7 +441,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])", + "expr": "rate(container_cpu_usage_seconds_total{namespace='$namespace', pod=~'$component.*', container!='POD'}[$__rate_interval])", "interval": "", "intervalFactor": 5, "legendFormat": "{{pod}}", @@ -537,7 +537,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}", + "expr": "container_memory_working_set_bytes{namespace='$namespace', pod=~'$component.*', container!='POD'}", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -632,14 +632,14 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", + "expr": "rate(container_network_receive_bytes_total{namespace='$namespace', pod=~'$component.*'}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "rx-{{pod}}", "refId": "A" }, { - "expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", + "expr": "rate(container_network_transmit_bytes_total{namespace='$namespace', pod=~'$component.*'}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "tx-{{pod}}", @@ -735,7 +735,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}", + "expr": "kubelet_volume_stats_available_bytes{namespace='$namespace', persistentvolumeclaim=~'$component.*'}", "legendFormat": "{{persistentvolumeclaim}}", "refId": "A" } @@ -829,7 +829,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])", + "expr": "rate(promtail_custom_bad_words_total{exported_namespace='$namespace', app=~'$component.*'}[$__rate_interval])", "interval": "", "legendFormat": "{{exported_pod}}", "refId": "A" @@ -938,7 +938,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "tempodb_work_queue_length{namespace='$namespace', job=~'$namespace/$component'} / tempodb_work_queue_max{namespace='$namespace', job=~'$namespace/$component'}", "legendFormat": "{{instance}}", "refId": "A" } @@ -1032,17 +1032,17 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "expr": "sum(increase(tempodb_compaction_errors_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (job)", "legendFormat": "compaction_err", "refId": "B" }, { - "expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "expr": "sum(increase(tempodb_retention_errors_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (job)", "legendFormat": "retention_err", "refId": "C" }, { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "expr": "sum(increase(tempodb_blocklist_poll_errors_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (job)", "legendFormat": "blocklist_err", "refId": "D" } @@ -1136,18 +1136,18 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (le))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1243,7 +1243,7 @@ "targets": [ { "exemplar": true, - "expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}) by (tenant)", + "expr": "avg(tempodb_blocklist_length{namespace='$namespace', job=~'$namespace/$component'}) by (tenant)", "instant": false, "interval": "", "legendFormat": "{{tenant}}", @@ -1339,19 +1339,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1446,13 +1446,13 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "expr": "sum(increase(tempodb_retention_deleted_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval]))", "interval": "", "legendFormat": "deleted", "refId": "A" }, { - "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval]))", "interval": "", "legendFormat": "marked_for_deletion", "refId": "B" @@ -1547,7 +1547,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "expr": "increase(kube_pod_container_status_restarts_total{namespace='$namespace'}[5m]) > 0", "hide": false, "interval": "", "legendFormat": "{{pod}}", @@ -1660,7 +1660,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid', namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1755,19 +1755,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1862,7 +1862,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*', namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1959,7 +1959,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -1967,7 +1967,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_search.*'}[$__rate_interval])) by (le))", "hide": false, "interval": "", "legendFormat": ".9", @@ -1976,7 +1976,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_search.*'}[$__rate_interval])) by (le))", "hide": false, "interval": "", "legendFormat": ".5", @@ -2072,7 +2072,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', route=~'.*api_traces_traceid', job='$namespace/query-frontend'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2167,19 +2167,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2274,7 +2274,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', route=~'.*api_search.*', job='$namespace/query-frontend'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2371,7 +2371,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2379,7 +2379,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2387,7 +2387,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -2482,7 +2482,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid', namespace='$namespace', job='$namespace/querier'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2577,19 +2577,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2684,7 +2684,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*', namespace='$namespace', job='$namespace/querier'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2781,7 +2781,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2789,7 +2789,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2797,7 +2797,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -2892,7 +2892,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID', namespace='$namespace', job='$namespace/ingester'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2987,19 +2987,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -3094,7 +3094,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*', namespace='$namespace', job='$namespace/ingester'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3191,7 +3191,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -3199,7 +3199,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -3207,7 +3207,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -3308,7 +3308,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_metrics_summary\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics_summary', namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3410,7 +3410,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3422,7 +3422,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3434,7 +3434,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3536,7 +3536,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_metrics.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', route=~'.*api_metrics.*', job='$namespace/query-frontend'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3640,7 +3640,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3654,7 +3654,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3668,7 +3668,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3770,7 +3770,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_metrics.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*', namespace='$namespace', job='$namespace/querier'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}} {{route}}", @@ -3872,7 +3872,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3884,7 +3884,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3896,7 +3896,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3997,7 +3997,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.MetricsGenerator/GetMetrics\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics', namespace='$namespace', job='$namespace/metrics-generator'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -4099,7 +4099,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -4111,7 +4111,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -4123,7 +4123,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/tempopb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -4243,7 +4243,7 @@ "uid": "P666011C0B63BDCA4" }, "editorMode": "code", - "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "{{pod}}", "range": true, @@ -4345,7 +4345,7 @@ "uid": "P1809F7CD0C75ACF3" }, "editorMode": "builder", - "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "", "range": true, @@ -4441,7 +4441,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1h]))", + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{namespace='$namespace', job='$namespace/ingester'}[1h]))", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -4536,7 +4536,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "increase(tempo_ingester_failed_flushes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "expr": "increase(tempo_ingester_failed_flushes_total{namespace='$namespace'}[5m]) > 0", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -4631,7 +4631,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{namespace='$namespace'}[1h]))", "interval": "", "legendFormat": "", "refId": "A" @@ -4726,19 +4726,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace='$namespace'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace='$namespace'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace='$namespace'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -4839,7 +4839,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "sum(tempo_distributor_queue_length{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\".*/distributor\"}) by (name)", + "expr": "sum(tempo_distributor_queue_length{cluster=~'$cluster', namespace=~'$namespace', job=~'.*/distributor'}) by (name)", "interval": "", "legendFormat": "{{name}}", "range": true, @@ -4935,7 +4935,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (status_code)", "interval": "", "legendFormat": "{{status_code}}", "refId": "A" @@ -5030,13 +5030,13 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_accepted_spans{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "accepted", "refId": "A" }, { - "expr": "sum(rate(tempo_receiver_refused_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_refused_spans{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "refused", "refId": "B" @@ -5131,19 +5131,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -5238,19 +5238,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -5351,7 +5351,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_metrics_generator_spans_received_total{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "accepted", "range": true, @@ -5363,7 +5363,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (reason)", + "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{namespace='$namespace'}[$__rate_interval])) by (reason)", "interval": "", "legendFormat": "refused {{reason}}", "range": true, @@ -5465,7 +5465,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -5477,7 +5477,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -5489,7 +5489,7 @@ "uid": "cortex-ops-01" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -5602,7 +5602,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_count{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (status_code, method)", "interval": "", "legendFormat": "{{status_code}}-{{method}}", "refId": "A" @@ -5696,19 +5696,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".99-{{method}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".9-{{method}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".5-{{method}}", "refId": "C" @@ -5820,7 +5820,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (status_code, operation)", "interval": "", "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5914,17 +5914,17 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (operation, le))", "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (operation, le))", "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (operation, le))", "legendFormat": ".5-{{operation}}", "refId": "C" } @@ -6040,7 +6040,7 @@ "type": "prometheus", "uid": "P666011C0B63BDCA4" }, - "expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "gauge_memberlist_health_score{namespace='$namespace', job=~'$namespace/$component'}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -6134,7 +6134,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "tempo_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "tempo_memberlist_client_cluster_node_health_score{namespace='$namespace', job=~'$namespace/$component'}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -6228,13 +6228,13 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "min(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "min(tempo_memberlist_client_cluster_members_count{namespace='$namespace', job=~'$namespace/$component'})", "interval": "", "legendFormat": "min", "refId": "A" }, { - "expr": "max(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "max(tempo_memberlist_client_cluster_members_count{namespace='$namespace', job=~'$namespace/$component'})", "interval": "", "legendFormat": "max", "refId": "B" @@ -6333,7 +6333,7 @@ "type": "prometheus", "uid": "P666011C0B63BDCA4" }, - "expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "min(tempo_memberlist_client_kv_store_count{namespace='$namespace', job=~'$namespace/$component'})", "interval": "", "legendFormat": "min", "refId": "A" @@ -6343,7 +6343,7 @@ "type": "prometheus", "uid": "P666011C0B63BDCA4" }, - "expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "max(tempo_memberlist_client_kv_store_count{namespace='$namespace', job=~'$namespace/$component'})", "interval": "", "legendFormat": "max", "refId": "B" @@ -6449,7 +6449,7 @@ "pluginVersion": "7.5.0-14141pre", "targets": [ { - "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_vulture_trace_error_total{namespace='$namespace'}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "{{error}}", "refId": "A" @@ -6513,7 +6513,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_vulture_trace_error_total{namespace='$namespace'}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{namespace='$namespace'}[$__rate_interval]))", "interval": "", "legendFormat": "{{secondsago}}", "refId": "A" @@ -6622,7 +6622,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_objects_combined_total{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -6714,7 +6714,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_objects_written_total{namespace='$namespace', job='$namespace/compactor'}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -6807,7 +6807,7 @@ "uid": "P666011C0B63BDCA4" }, "editorMode": "builder", - "expr": "sum(rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_bytes_written_total{namespace='$namespace', job='$namespace/compactor'}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "__auto", "range": true, @@ -6901,7 +6901,7 @@ "uid": "P666011C0B63BDCA4" }, "editorMode": "code", - "expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[5m])) by (level)", + "expr": "sum(increase(tempodb_compaction_blocks_total{namespace='$namespace', job='$namespace/compactor'}[5m])) by (level)", "interval": "", "legendFormat": "__auto", "range": true, diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json index 58630b2..9636108 100644 --- a/src/grafana_dashboards/tempo-reads.json +++ b/src/grafana_dashboards/tempo-reads.json @@ -74,7 +74,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -159,7 +159,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -168,7 +168,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -177,7 +177,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -284,7 +284,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/query-frontend', route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -369,7 +369,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -378,7 +378,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -387,7 +387,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -494,7 +494,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/querier', route=~'querier_api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -579,7 +579,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -588,7 +588,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -597,7 +597,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -704,7 +704,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{job=~'($namespace)/querier'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -789,7 +789,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (le,endpoint)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{job=~'($namespace)/querier'}[$__rate_interval])) by (le,endpoint)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -798,7 +798,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (le,endpoint)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{job=~'($namespace)/querier'}[$__rate_interval])) by (le,endpoint)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -807,7 +807,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (endpoint)", + "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{job=~'($namespace)/querier'}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{job=~'($namespace)/querier'}[$__rate_interval])) by (endpoint)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -914,7 +914,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/tempopb.Querier/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester', route=~'/tempopb.Querier/.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -999,7 +999,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1008,7 +1008,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1017,7 +1017,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1124,7 +1124,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1209,7 +1209,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1218,7 +1218,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1227,7 +1227,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1334,7 +1334,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/querier',operation='GET'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1419,7 +1419,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1428,7 +1428,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1437,7 +1437,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/src/grafana_dashboards/tempo-resources.json b/src/grafana_dashboards/tempo-resources.json index 44e353a..da78786 100644 --- a/src/grafana_dashboards/tempo-resources.json +++ b/src/grafana_dashboards/tempo-resources.json @@ -75,21 +75,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'cortex-gw(-internal)?'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'cortex-gw(-internal)?'} / container_spec_cpu_period{namespace=~'$namespace',container=~'cortex-gw(-internal)?'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'cortex-gw(-internal)?', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -183,21 +183,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'cortex-gw(-internal)?'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'cortex-gw(-internal)?'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'cortex-gw(-internal)?', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -282,7 +282,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/cortex-gw(-internal)?'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -388,21 +388,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'distributor'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'distributor'} / container_spec_cpu_period{namespace=~'$namespace',container=~'distributor'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'distributor', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -496,21 +496,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'distributor'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'distributor'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'distributor', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -595,7 +595,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/distributor'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -701,21 +701,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'ingester'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'ingester'} / container_spec_cpu_period{namespace=~'$namespace',container=~'ingester'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'ingester', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -809,21 +809,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'ingester'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'ingester'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'ingester', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -908,7 +908,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/ingester'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1014,21 +1014,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'metrics-generator'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'metrics-generator'} / container_spec_cpu_period{namespace=~'$namespace',container=~'metrics-generator'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'metrics-generator', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1122,21 +1122,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'metrics-generator'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'metrics-generator'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'metrics-generator', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1221,7 +1221,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/metrics-generator'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1327,21 +1327,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'query-frontend'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'query-frontend'} / container_spec_cpu_period{namespace=~'$namespace',container=~'query-frontend'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'query-frontend', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1435,21 +1435,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'query-frontend'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'query-frontend'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'query-frontend', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1534,7 +1534,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/query-frontend'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1640,21 +1640,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'querier'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'querier'} / container_spec_cpu_period{namespace=~'$namespace',container=~'querier'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'querier', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1748,21 +1748,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'querier'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'querier'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'querier', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1847,7 +1847,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/querier'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1953,21 +1953,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'compactor'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})", + "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'compactor'} / container_spec_cpu_period{namespace=~'$namespace',container=~'compactor'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'compactor', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -2061,21 +2061,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})", + "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'compactor'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'compactor'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'compactor', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -2160,7 +2160,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/compactor'})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", diff --git a/src/grafana_dashboards/tempo-rollout-progress.json b/src/grafana_dashboards/tempo-rollout-progress.json index 37a1313..fb627be 100644 --- a/src/grafana_dashboards/tempo-rollout-progress.json +++ b/src/grafana_dashboards/tempo-rollout-progress.json @@ -108,7 +108,7 @@ "steppedLine": false, "targets": [ { - "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", + "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{namespace=~'$namespace',statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", "format": null, "intervalFactor": null, "legendFormat": "{{tempo_service}}", @@ -116,7 +116,7 @@ "step": null }, { - "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"tempo_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", + "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{namespace=~'$namespace',deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", "format": null, "intervalFactor": null, "legendFormat": "{{tempo_service}}", @@ -230,7 +230,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -354,7 +354,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -474,7 +474,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -598,7 +598,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))\n", + "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))\n", "format": null, "instant": false, "interval": "", @@ -714,7 +714,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -838,7 +838,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -958,7 +958,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -1082,7 +1082,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))\n", + "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}))\n", "format": null, "instant": false, "interval": "", @@ -1212,7 +1212,7 @@ "steppedLine": false, "targets": [ { - "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n", + "expr": "kube_deployment_status_replicas_unavailable{namespace=~'$namespace', deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", "format": null, "instant": true, "interval": "", @@ -1222,7 +1222,7 @@ "step": null }, { - "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"}\n> 0\n", + "expr": "kube_statefulset_status_replicas_current{namespace=~'$namespace', statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'} -\nkube_statefulset_status_replicas_ready {namespace=~'$namespace', statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", "format": null, "instant": true, "interval": "", @@ -1299,7 +1299,7 @@ "id": 11, "targets": [ { - "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\".*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*\"},\n \"version\", \"$1\", \"image\", \".*:(.+)-.*\"\n )\n)\n", + "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{namespace=~'$namespace',container=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'version', '$1', 'image', '.*:(.+)-.*'\n )\n)\n", "instant": true, "legendFormat": "", "refId": "A" @@ -1381,14 +1381,14 @@ "steppedLine": false, "targets": [ { - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))[1h:])\n)\n", + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))[1h:])\n)\n", "format": "time_series", "interval": "1m", "legendFormat": "writes", "legendLink": null }, { - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"tempo_api_.*\"}))[1h:])\n)\n", + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}))[1h:])\n)\n", "format": "time_series", "interval": "1m", "legendFormat": "reads", diff --git a/src/grafana_dashboards/tempo-tenants.json b/src/grafana_dashboards/tempo-tenants.json index 384bc31..99622c9 100644 --- a/src/grafana_dashboards/tempo-tenants.json +++ b/src/grafana_dashboards/tempo-tenants.json @@ -89,7 +89,7 @@ ], "targets": [ { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n) by (limit_name)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor'})\n) by (limit_name)\n", "format": "table", "instant": true, "legendFormat": "", @@ -196,21 +196,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_distributor_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_distributor_bytes_received_total{job=~'($namespace)/distributor',tenant='$tenant'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "received", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_rate_limit_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_rate_limit_bytes\"})\n) by (ingestion_rate_limit_bytes)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='ingestion_rate_limit_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='ingestion_rate_limit_bytes'})\n) by (ingestion_rate_limit_bytes)\n", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_burst_size_bytes\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_burst_size_bytes\"})\n) by (ingestion_burst_size_bytes)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='ingestion_burst_size_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='ingestion_burst_size_bytes'})\n) by (ingestion_burst_size_bytes)\n", "format": "time_series", "interval": "1m", "legendFormat": "burst limit", @@ -295,14 +295,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_distributor_spans_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_distributor_spans_received_total{job=~'($namespace)/distributor',tenant='$tenant'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "accepted", "legendLink": null }, { - "expr": "sum(rate(tempo_discarded_spans_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval])) by (reason)", + "expr": "sum(rate(tempo_discarded_spans_total{job=~'($namespace)/distributor',tenant='$tenant'}[$__rate_interval])) by (reason)", "format": "time_series", "interval": "1m", "legendFormat": "refused {{ reason }}", @@ -396,21 +396,21 @@ "steppedLine": false, "targets": [ { - "expr": "max(tempo_ingester_live_traces{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",tenant=\"$tenant\"})", + "expr": "max(tempo_ingester_live_traces{job=~'($namespace)/ingester',tenant='$tenant'})", "format": "time_series", "interval": "1m", "legendFormat": "live traces", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_global_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_global_traces_per_user\"})\n) by (max_global_traces_per_user)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='max_global_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='max_global_traces_per_user'})\n) by (max_global_traces_per_user)\n", "format": "time_series", "interval": "1m", "legendFormat": "global limit", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_local_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_local_traces_per_user\"})\n) by (max_local_traces_per_user)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='max_local_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='max_local_traces_per_user'})\n) by (max_local_traces_per_user)\n", "format": "time_series", "interval": "1m", "legendFormat": "local limit", @@ -507,7 +507,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"traces\"}[$__rate_interval])) by (status)", + "expr": "sum(rate(tempo_query_frontend_queries_total{job=~'($namespace)/query-frontend',tenant='$tenant',op='traces'}[$__rate_interval])) by (status)", "format": "time_series", "interval": "1m", "legendFormat": "{{ status }}", @@ -592,7 +592,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"search\"}[$__rate_interval])) by (status)", + "expr": "sum(rate(tempo_query_frontend_queries_total{job=~'($namespace)/query-frontend',tenant='$tenant',op='search'}[$__rate_interval])) by (status)", "format": "time_series", "interval": "1m", "legendFormat": "{{ status }}", @@ -683,7 +683,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})", + "expr": "avg(tempodb_blocklist_length{job=~'($namespace)/compactor',tenant='$tenant'})", "format": "time_series", "interval": "1m", "legendFormat": "length", @@ -762,7 +762,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tempodb_compaction_outstanding_blocks{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})\n/\ncount(tempo_build_info{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n", + "expr": "sum(tempodb_compaction_outstanding_blocks{job=~'($namespace)/compactor',tenant='$tenant'})\n/\ncount(tempo_build_info{job=~'($namespace)/compactor'})\n", "format": "time_series", "interval": "1m", "legendFormat": "blocks", @@ -853,7 +853,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{job=~'($namespace)/metrics-generator',tenant='$tenant'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "rate", @@ -942,14 +942,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tempo_metrics_generator_registry_active_series{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"})", + "expr": "sum(tempo_metrics_generator_registry_active_series{job=~'($namespace)/metrics-generator',tenant='$tenant'})", "format": "time_series", "interval": "1m", "legendFormat": "{{ tenant }}", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"metrics_generator_max_active_series\"})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"metrics_generator_max_active_series\"})\n) by (metrics_generator_max_active_series)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='metrics_generator_max_active_series'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='metrics_generator_max_active_series'})\n) by (metrics_generator_max_active_series)\n", "format": "time_series", "interval": "1m", "legendFormat": "limit", @@ -1099,7 +1099,7 @@ "options": [ ], - "query": "label_values(tempodb_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"}, tenant)", + "query": "label_values(tempodb_blocklist_length{job=~'($namespace)/compactor'}, tenant)", "refresh": 1, "regex": "", "sort": 2, diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json index d7c128b..e5056ab 100644 --- a/src/grafana_dashboards/tempo-writes.json +++ b/src/grafana_dashboards/tempo-writes.json @@ -74,7 +74,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -159,7 +159,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -168,7 +168,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -177,7 +177,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -276,7 +276,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", __name__=~\"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+\"},\n \"grpc_status\", \"$1\", \"__name__\", \"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)\"\n )\n [$__interval:$__interval]\n )\n)\n", + "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {job=~'($namespace)/cortex-gw(-internal)?', __name__=~'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+'},\n 'grpc_status', '$1', '__name__', 'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)'\n )\n [$__interval:$__interval]\n )\n)\n", "format": "time_series", "interval": "1m", "legendFormat": "{{grpc_status}}", @@ -337,7 +337,7 @@ }, "id": 4, "options": { - "content": "Visit [Status codes and their use in gRPC](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md)\n\nCode | Number | Description\n---|---|---\nOK | 0 | Not an error; returned on success.\nCANCELLED | 1 | The operation was cancelled, typically by the caller.\nUNKNOWN | 2 | Unknown error. For example, this error may be returned when a Status value received from another address space belongs to an error space that is not known in this address space. Also errors raised by APIs that do not return enough error information may be converted to this error.\nINVALID_ARGUMENT | 3 | The client specified an invalid argument. Note that this differs from FAILED_PRECONDITION. INVALID_ARGUMENT indicates arguments that are problematic regardless of the state of the system (e.g., a malformed file name).\nDEADLINE_EXCEEDED | 4 | The deadline expired before the operation could complete. For operations that change the state of the system, this error may be returned even if the operation has completed successfully. For example, a successful response from a server could have been delayed long\nNOT_FOUND | 5 | Some requested entity (e.g., file or directory) was not found. Note to server developers: if a request is denied for an entire class of users, such as gradual feature rollout or undocumented allowlist, NOT_FOUND may be used. If a request is denied for some users within a class of users, such as user-based access control, PERMISSION_DENIED must be used.\nALREADY_EXISTS | 6 | The entity that a client attempted to create (e.g., file or directory) already exists.\nPERMISSION_DENIED | 7 | The caller does not have permission to execute the specified operation. PERMISSION_DENIED must not be used for rejections caused by exhausting some resource (use RESOURCE_EXHAUSTED instead for those errors). PERMISSION_DENIED must not be used if the caller can not be identified (use UNAUTHENTICATED instead for those errors). This error code does not imply the request is valid or the requested entity exists or satisfies other pre-conditions.\nRESOURCE_EXHAUSTED | 8 | Some resource has been exhausted, perhaps a per-user quota, or perhaps the entire file system is out of space.\nFAILED_PRECONDITION | 9 | The operation was rejected because the system is not in a state required for the operation's execution. For example, the directory to be deleted is non-empty, an rmdir operation is applied to a non-directory, etc. Service implementors can use the following guidelines to decide between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE: (a) Use UNAVAILABLE if the client can retry just the failing call. (b) Use ABORTED if the client should retry at a higher level (e.g., when a client-specified test-and-set fails, indicating the client should restart a read-modify-write sequence). (c) Use FAILED_PRECONDITION if the client should not retry until the system state has been explicitly fixed. E.g., if an \"rmdir\" fails because the directory is non-empty, FAILED_PRECONDITION should be returned since the client should not retry unless the files are deleted from the directory.\nABORTED | 10 | The operation was aborted, typically due to a concurrency issue such as a sequencer check failure or transaction abort. See the guidelines above for deciding between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE.\nOUT_OF_RANGE | 11 | The operation was attempted past the valid range. E.g., seeking or reading past end-of-file. Unlike INVALID_ARGUMENT, this error indicates a problem that may be fixed if the system state changes. For example, a 32-bit file system will generate INVALID_ARGUMENT if asked to read at an offset that is not in the range [0,2^32-1], but it will generate OUT_OF_RANGE if asked to read from an offset past the current file size. There is a fair bit of overlap between FAILED_PRECONDITION and OUT_OF_RANGE. We recommend using OUT_OF_RANGE (the more specific error) when it applies so that callers who are iterating through a space can easily look for an OUT_OF_RANGE error to detect when they are done.\nUNIMPLEMENTED | 12 | The operation is not implemented or is not supported/enabled in this service.\nINTERNAL | 13 | Internal errors. This means that some invariants expected by the underlying system have been broken. This error code is reserved for serious errors.\nUNAVAILABLE | 14 | The service is currently unavailable. This is most likely a transient condition, which can be corrected by retrying with a backoff. Note that it is not always safe to retry non-idempotent operations.\nDATA_LOSS | 15 | Unrecoverable data loss or corruption.\nUNAUTHENTICATED | 16 | The request does not have valid authentication credentials for the operation.\n", + "content": "Visit [Status codes and their use in gRPC](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md)\n\nCode | Number | Description\n---|---|---\nOK | 0 | Not an error; returned on success.\nCANCELLED | 1 | The operation was cancelled, typically by the caller.\nUNKNOWN | 2 | Unknown error. For example, this error may be returned when a Status value received from another address space belongs to an error space that is not known in this address space. Also errors raised by APIs that do not return enough error information may be converted to this error.\nINVALID_ARGUMENT | 3 | The client specified an invalid argument. Note that this differs from FAILED_PRECONDITION. INVALID_ARGUMENT indicates arguments that are problematic regardless of the state of the system (e.g., a malformed file name).\nDEADLINE_EXCEEDED | 4 | The deadline expired before the operation could complete. For operations that change the state of the system, this error may be returned even if the operation has completed successfully. For example, a successful response from a server could have been delayed long\nNOT_FOUND | 5 | Some requested entity (e.g., file or directory) was not found. Note to server developers: if a request is denied for an entire class of users, such as gradual feature rollout or undocumented allowlist, NOT_FOUND may be used. If a request is denied for some users within a class of users, such as user-based access control, PERMISSION_DENIED must be used.\nALREADY_EXISTS | 6 | The entity that a client attempted to create (e.g., file or directory) already exists.\nPERMISSION_DENIED | 7 | The caller does not have permission to execute the specified operation. PERMISSION_DENIED must not be used for rejections caused by exhausting some resource (use RESOURCE_EXHAUSTED instead for those errors). PERMISSION_DENIED must not be used if the caller can not be identified (use UNAUTHENTICATED instead for those errors). This error code does not imply the request is valid or the requested entity exists or satisfies other pre-conditions.\nRESOURCE_EXHAUSTED | 8 | Some resource has been exhausted, perhaps a per-user quota, or perhaps the entire file system is out of space.\nFAILED_PRECONDITION | 9 | The operation was rejected because the system is not in a state required for the operation's execution. For example, the directory to be deleted is non-empty, an rmdir operation is applied to a non-directory, etc. Service implementors can use the following guidelines to decide between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE: (a) Use UNAVAILABLE if the client can retry just the failing call. (b) Use ABORTED if the client should retry at a higher level (e.g., when a client-specified test-and-set fails, indicating the client should restart a read-modify-write sequence). (c) Use FAILED_PRECONDITION if the client should not retry until the system state has been explicitly fixed. E.g., if an 'rmdir' fails because the directory is non-empty, FAILED_PRECONDITION should be returned since the client should not retry unless the files are deleted from the directory.\nABORTED | 10 | The operation was aborted, typically due to a concurrency issue such as a sequencer check failure or transaction abort. See the guidelines above for deciding between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE.\nOUT_OF_RANGE | 11 | The operation was attempted past the valid range. E.g., seeking or reading past end-of-file. Unlike INVALID_ARGUMENT, this error indicates a problem that may be fixed if the system state changes. For example, a 32-bit file system will generate INVALID_ARGUMENT if asked to read at an offset that is not in the range [0,2^32-1], but it will generate OUT_OF_RANGE if asked to read from an offset past the current file size. There is a fair bit of overlap between FAILED_PRECONDITION and OUT_OF_RANGE. We recommend using OUT_OF_RANGE (the more specific error) when it applies so that callers who are iterating through a space can easily look for an OUT_OF_RANGE error to detect when they are done.\nUNIMPLEMENTED | 12 | The operation is not implemented or is not supported/enabled in this service.\nINTERNAL | 13 | Internal errors. This means that some invariants expected by the underlying system have been broken. This error code is reserved for serious errors.\nUNAVAILABLE | 14 | The service is currently unavailable. This is most likely a transient condition, which can be corrected by retrying with a backoff. Note that it is not always safe to retry non-idempotent operations.\nDATA_LOSS | 15 | Unrecoverable data loss or corruption.\nUNAUTHENTICATED | 16 | The request does not have valid authentication credentials for the operation.\n", "mode": "markdown" }, "span": 6, @@ -397,14 +397,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_accepted_spans{job=~'($namespace)/distributor'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "accepted", "legendLink": null }, { - "expr": "sum(rate(tempo_receiver_refused_spans{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_refused_spans{job=~'($namespace)/distributor'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "refused", @@ -489,7 +489,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{job=~'($namespace)/distributor'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -498,7 +498,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{job=~'($namespace)/distributor'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -507,7 +507,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{job=~'($namespace)/distributor'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{job=~'($namespace)/distributor'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -614,7 +614,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -699,7 +699,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -708,7 +708,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -717,7 +717,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -824,7 +824,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -909,7 +909,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -918,7 +918,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -927,7 +927,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1034,7 +1034,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1119,7 +1119,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1128,7 +1128,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1137,7 +1137,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1244,7 +1244,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1329,7 +1329,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1338,7 +1338,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1347,7 +1347,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1454,7 +1454,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1539,7 +1539,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1548,7 +1548,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1557,7 +1557,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, From 0da1bb9fcc9f8a3652132c472f1f143830047607 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Fri, 16 Aug 2024 15:48:14 +0200 Subject: [PATCH 04/14] Dashboards load and mostly show something --- src/grafana_dashboards/tempo-operational.json | 306 +++++++++--------- src/grafana_dashboards/tempo-reads.json | 56 ++-- src/grafana_dashboards/tempo-resources.json | 98 +++--- .../tempo-rollout-progress.json | 32 +- src/grafana_dashboards/tempo-tenants.json | 36 +-- src/grafana_dashboards/tempo-writes.json | 62 ++-- src/grafana_dashboards/tempo.json | 2 +- 7 files changed, 296 insertions(+), 296 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index 60b6923..7e0e8ba 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -26,7 +26,7 @@ "uid": "$logsds" }, "enable": true, - "expr": "{diff_namespace='$namespace', container='kube-diff-logger'}", + "expr": "{diff_{}container='kube-diff-logger'}", "hide": true, "iconColor": "rgba(255, 96, 96, 1)", "name": "diffs", @@ -60,7 +60,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -161,7 +161,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(go_gc_duration_seconds_count{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])", + "expr": "rate(go_gc_duration_seconds_count{}[$__rate_interval])", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -256,7 +256,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{namespace='$namespace', job=~'$namespace/$component'}", + "expr": "go_memstats_heap_inuse_bytes{}", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -351,7 +351,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "go_goroutines{namespace='$namespace', job=~'$namespace/$component'}", + "expr": "go_goroutines{}", "legendFormat": "{{pod}}", "refId": "A" } @@ -441,7 +441,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(container_cpu_usage_seconds_total{namespace='$namespace', pod=~'$component.*', container!='POD'}[$__rate_interval])", + "expr": "rate(container_cpu_usage_seconds_total{pod=~'$component.*', container!='POD'}[$__rate_interval])", "interval": "", "intervalFactor": 5, "legendFormat": "{{pod}}", @@ -537,7 +537,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "container_memory_working_set_bytes{namespace='$namespace', pod=~'$component.*', container!='POD'}", + "expr": "container_memory_working_set_bytes{pod=~'$component.*', container!='POD'}", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -632,14 +632,14 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(container_network_receive_bytes_total{namespace='$namespace', pod=~'$component.*'}[$__rate_interval])", + "expr": "rate(container_network_receive_bytes_total{pod=~'$component.*'}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "rx-{{pod}}", "refId": "A" }, { - "expr": "rate(container_network_transmit_bytes_total{namespace='$namespace', pod=~'$component.*'}[$__rate_interval])", + "expr": "rate(container_network_transmit_bytes_total{pod=~'$component.*'}[$__rate_interval])", "hide": false, "interval": "", "legendFormat": "tx-{{pod}}", @@ -735,7 +735,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "kubelet_volume_stats_available_bytes{namespace='$namespace', persistentvolumeclaim=~'$component.*'}", + "expr": "kubelet_volume_stats_available_bytes{persistentvolumeclaim=~'$component.*'}", "legendFormat": "{{persistentvolumeclaim}}", "refId": "A" } @@ -829,7 +829,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "rate(promtail_custom_bad_words_total{exported_namespace='$namespace', app=~'$component.*'}[$__rate_interval])", + "expr": "rate(promtail_custom_bad_words_total{app=~'$component.*'}[$__rate_interval])", "interval": "", "legendFormat": "{{exported_pod}}", "refId": "A" @@ -842,7 +842,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -938,7 +938,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "tempodb_work_queue_length{namespace='$namespace', job=~'$namespace/$component'} / tempodb_work_queue_max{namespace='$namespace', job=~'$namespace/$component'}", + "expr": "tempodb_work_queue_length{} / tempodb_work_queue_max{}", "legendFormat": "{{instance}}", "refId": "A" } @@ -1032,17 +1032,17 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_compaction_errors_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (job)", + "expr": "sum(increase(tempodb_compaction_errors_total{}[$__rate_interval])) by (job)", "legendFormat": "compaction_err", "refId": "B" }, { - "expr": "sum(increase(tempodb_retention_errors_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (job)", + "expr": "sum(increase(tempodb_retention_errors_total{}[$__rate_interval])) by (job)", "legendFormat": "retention_err", "refId": "C" }, { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (job)", + "expr": "sum(increase(tempodb_blocklist_poll_errors_total{}[$__rate_interval])) by (job)", "legendFormat": "blocklist_err", "refId": "D" } @@ -1136,18 +1136,18 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1243,7 +1243,7 @@ "targets": [ { "exemplar": true, - "expr": "avg(tempodb_blocklist_length{namespace='$namespace', job=~'$namespace/$component'}) by (tenant)", + "expr": "avg(tempodb_blocklist_length{}) by (tenant)", "instant": false, "interval": "", "legendFormat": "{{tenant}}", @@ -1339,19 +1339,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1446,13 +1446,13 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_retention_deleted_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval]))", + "expr": "sum(increase(tempodb_retention_deleted_total{}[$__rate_interval]))", "interval": "", "legendFormat": "deleted", "refId": "A" }, { - "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval]))", + "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{}[$__rate_interval]))", "interval": "", "legendFormat": "marked_for_deletion", "refId": "B" @@ -1547,7 +1547,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{namespace='$namespace'}[5m]) > 0", + "expr": "increase(kube_pod_container_status_restarts_total{}[5m]) > 0", "hide": false, "interval": "", "legendFormat": "{{pod}}", @@ -1565,7 +1565,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -1660,7 +1660,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid', namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1755,19 +1755,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1862,7 +1862,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*', namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1959,7 +1959,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -1967,7 +1967,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", "hide": false, "interval": "", "legendFormat": ".9", @@ -1976,7 +1976,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", "hide": false, "interval": "", "legendFormat": ".5", @@ -2072,7 +2072,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', route=~'.*api_traces_traceid', job='$namespace/query-frontend'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2167,19 +2167,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2274,7 +2274,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', route=~'.*api_search.*', job='$namespace/query-frontend'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2371,7 +2371,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2379,7 +2379,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2387,7 +2387,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -2482,7 +2482,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid', namespace='$namespace', job='$namespace/querier'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2577,19 +2577,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2684,7 +2684,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*', namespace='$namespace', job='$namespace/querier'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2781,7 +2781,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2789,7 +2789,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2797,7 +2797,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -2892,7 +2892,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID', namespace='$namespace', job='$namespace/ingester'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2987,19 +2987,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -3094,7 +3094,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*', namespace='$namespace', job='$namespace/ingester'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3191,7 +3191,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -3199,7 +3199,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -3207,7 +3207,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -3305,10 +3305,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics_summary', namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics_summary'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3407,10 +3407,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3419,10 +3419,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3431,10 +3431,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3532,11 +3532,11 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', route=~'.*api_metrics.*', job='$namespace/query-frontend'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3636,11 +3636,11 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3650,11 +3650,11 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3664,11 +3664,11 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/query-frontend', route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3767,10 +3767,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*', namespace='$namespace', job='$namespace/querier'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}} {{route}}", @@ -3869,10 +3869,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3881,10 +3881,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3893,10 +3893,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/querier', route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3994,10 +3994,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics', namespace='$namespace', job='$namespace/metrics-generator'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -4096,10 +4096,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -4108,10 +4108,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -4120,10 +4120,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -4141,7 +4141,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -4240,10 +4240,10 @@ { "datasource": { "type": "prometheus", - "uid": "P666011C0B63BDCA4" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", "interval": "", "legendFormat": "{{pod}}", "range": true, @@ -4342,10 +4342,10 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "editorMode": "builder", - "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", "interval": "", "legendFormat": "", "range": true, @@ -4441,7 +4441,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{namespace='$namespace', job='$namespace/ingester'}[1h]))", + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{job='$namespace/ingester'}[1h]))", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -4536,7 +4536,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "increase(tempo_ingester_failed_flushes_total{namespace='$namespace'}[5m]) > 0", + "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", "interval": "", "legendFormat": "{{pod}}", "refId": "A" @@ -4631,7 +4631,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{namespace='$namespace'}[1h]))", + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", "interval": "", "legendFormat": "", "refId": "A" @@ -4726,19 +4726,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace='$namespace'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace='$namespace'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace='$namespace'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -4836,10 +4836,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(tempo_distributor_queue_length{cluster=~'$cluster', namespace=~'$namespace', job=~'.*/distributor'}) by (name)", + "expr": "sum(tempo_distributor_queue_length{cluster=~'$cluster'}) by (name)", "interval": "", "legendFormat": "{{name}}", "range": true, @@ -4935,7 +4935,7 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (status_code)", "interval": "", "legendFormat": "{{status_code}}", "refId": "A" @@ -5030,13 +5030,13 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_receiver_accepted_spans{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", "interval": "", "legendFormat": "accepted", "refId": "A" }, { - "expr": "sum(rate(tempo_receiver_refused_spans{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", "interval": "", "legendFormat": "refused", "refId": "B" @@ -5131,19 +5131,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -5238,19 +5238,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -5348,10 +5348,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_received_total{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", "interval": "", "legendFormat": "accepted", "range": true, @@ -5360,10 +5360,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{namespace='$namespace'}[$__rate_interval])) by (reason)", + "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", "interval": "", "legendFormat": "refused {{reason}}", "range": true, @@ -5462,10 +5462,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -5474,10 +5474,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -5486,10 +5486,10 @@ { "datasource": { "type": "prometheus", - "uid": "cortex-ops-01" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace='$namespace', job='$namespace/metrics-generator', route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -5507,7 +5507,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -5602,7 +5602,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_count{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (status_code, method)", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_count{}[$__rate_interval])) by (status_code, method)", "interval": "", "legendFormat": "{{status_code}}-{{method}}", "refId": "A" @@ -5696,19 +5696,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".99-{{method}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".9-{{method}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".5-{{method}}", "refId": "C" @@ -5725,7 +5725,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -5820,7 +5820,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{}[$__rate_interval])) by (status_code, operation)", "interval": "", "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5914,17 +5914,17 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace='$namespace', job=~'$namespace/$component'}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", "legendFormat": ".5-{{operation}}", "refId": "C" } @@ -5940,7 +5940,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -6038,9 +6038,9 @@ { "datasource": { "type": "prometheus", - "uid": "P666011C0B63BDCA4" + "uid": "${prometheusds}" }, - "expr": "gauge_memberlist_health_score{namespace='$namespace', job=~'$namespace/$component'}", + "expr": "gauge_memberlist_health_score{}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -6134,7 +6134,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "tempo_memberlist_client_cluster_node_health_score{namespace='$namespace', job=~'$namespace/$component'}", + "expr": "tempo_memberlist_client_cluster_node_health_score{}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -6228,13 +6228,13 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "min(tempo_memberlist_client_cluster_members_count{namespace='$namespace', job=~'$namespace/$component'})", + "expr": "min(tempo_memberlist_client_cluster_members_count{})", "interval": "", "legendFormat": "min", "refId": "A" }, { - "expr": "max(tempo_memberlist_client_cluster_members_count{namespace='$namespace', job=~'$namespace/$component'})", + "expr": "max(tempo_memberlist_client_cluster_members_count{})", "interval": "", "legendFormat": "max", "refId": "B" @@ -6331,9 +6331,9 @@ { "datasource": { "type": "prometheus", - "uid": "P666011C0B63BDCA4" + "uid": "${prometheusds}" }, - "expr": "min(tempo_memberlist_client_kv_store_count{namespace='$namespace', job=~'$namespace/$component'})", + "expr": "min(tempo_memberlist_client_kv_store_count{})", "interval": "", "legendFormat": "min", "refId": "A" @@ -6341,9 +6341,9 @@ { "datasource": { "type": "prometheus", - "uid": "P666011C0B63BDCA4" + "uid": "${prometheusds}" }, - "expr": "max(tempo_memberlist_client_kv_store_count{namespace='$namespace', job=~'$namespace/$component'})", + "expr": "max(tempo_memberlist_client_kv_store_count{})", "interval": "", "legendFormat": "max", "refId": "B" @@ -6360,7 +6360,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -6449,7 +6449,7 @@ "pluginVersion": "7.5.0-14141pre", "targets": [ { - "expr": "sum(rate(tempo_vulture_trace_error_total{namespace='$namespace'}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum(rate(tempo_vulture_trace_error_total{}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{}[$__rate_interval]))", "interval": "", "legendFormat": "{{error}}", "refId": "A" @@ -6513,7 +6513,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_vulture_trace_error_total{namespace='$namespace'}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{namespace='$namespace'}[$__rate_interval]))", + "expr": "sum(rate(tempo_vulture_trace_error_total{}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{}[$__rate_interval]))", "interval": "", "legendFormat": "{{secondsago}}", "refId": "A" @@ -6530,7 +6530,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${prometheusds}" }, "gridPos": { "h": 1, @@ -6622,7 +6622,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempodb_compaction_objects_combined_total{namespace='$namespace', job=~'$namespace/compactor'}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_objects_combined_total{}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -6714,7 +6714,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempodb_compaction_objects_written_total{namespace='$namespace', job='$namespace/compactor'}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_objects_written_total{}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -6804,10 +6804,10 @@ { "datasource": { "type": "prometheus", - "uid": "P666011C0B63BDCA4" + "uid": "${prometheusds}" }, "editorMode": "builder", - "expr": "sum(rate(tempodb_compaction_bytes_written_total{namespace='$namespace', job='$namespace/compactor'}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_bytes_written_total{}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "__auto", "range": true, @@ -6898,10 +6898,10 @@ { "datasource": { "type": "prometheus", - "uid": "P666011C0B63BDCA4" + "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(increase(tempodb_compaction_blocks_total{namespace='$namespace', job='$namespace/compactor'}[5m])) by (level)", + "expr": "sum(increase(tempodb_compaction_blocks_total{}[5m])) by (level)", "interval": "", "legendFormat": "__auto", "range": true, diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json index 9636108..7d79f12 100644 --- a/src/grafana_dashboards/tempo-reads.json +++ b/src/grafana_dashboards/tempo-reads.json @@ -74,7 +74,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -159,7 +159,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -168,7 +168,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -177,7 +177,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?',route=~'api_.*'}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -284,7 +284,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/query-frontend', route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -369,7 +369,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -378,7 +378,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -387,7 +387,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/query-frontend',route=~'api_.*'}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -494,7 +494,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/querier', route=~'querier_api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -579,7 +579,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -588,7 +588,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -597,7 +597,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/querier',route=~'querier_api_.*'}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'querier_api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -704,7 +704,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{job=~'($namespace)/querier'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -789,7 +789,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{job=~'($namespace)/querier'}[$__rate_interval])) by (le,endpoint)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -798,7 +798,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{job=~'($namespace)/querier'}[$__rate_interval])) by (le,endpoint)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -807,7 +807,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{job=~'($namespace)/querier'}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{job=~'($namespace)/querier'}[$__rate_interval])) by (endpoint)", + "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval])) by (endpoint)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -914,7 +914,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester', route=~'/tempopb.Querier/.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -999,7 +999,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1008,7 +1008,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1017,7 +1017,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester',route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route)", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1124,7 +1124,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1209,7 +1209,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1218,7 +1218,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1227,7 +1227,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/querier',method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1334,7 +1334,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/querier',operation='GET'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1419,7 +1419,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1428,7 +1428,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1437,7 +1437,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/querier',operation='GET'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation='GET'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/src/grafana_dashboards/tempo-resources.json b/src/grafana_dashboards/tempo-resources.json index da78786..0ceceed 100644 --- a/src/grafana_dashboards/tempo-resources.json +++ b/src/grafana_dashboards/tempo-resources.json @@ -75,21 +75,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'cortex-gw(-internal)?'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'cortex-gw(-internal)?'} / container_spec_cpu_period{namespace=~'$namespace',container=~'cortex-gw(-internal)?'})", + "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'cortex-gw(-internal)?', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -183,21 +183,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'cortex-gw(-internal)?'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'cortex-gw(-internal)?'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'cortex-gw(-internal)?', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -282,7 +282,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/cortex-gw(-internal)?'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -388,21 +388,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'distributor'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'distributor'} / container_spec_cpu_period{namespace=~'$namespace',container=~'distributor'})", + "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'distributor', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -496,21 +496,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'distributor'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'distributor'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'distributor', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -595,7 +595,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/distributor'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -701,21 +701,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'ingester'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'ingester'} / container_spec_cpu_period{namespace=~'$namespace',container=~'ingester'})", + "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'ingester', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -809,21 +809,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'ingester'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'ingester'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'ingester', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -908,7 +908,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/ingester'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1014,21 +1014,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'metrics-generator'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'metrics-generator'} / container_spec_cpu_period{namespace=~'$namespace',container=~'metrics-generator'})", + "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'metrics-generator', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1122,21 +1122,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'metrics-generator'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'metrics-generator'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'metrics-generator', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1221,7 +1221,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/metrics-generator'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1327,21 +1327,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'query-frontend'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'query-frontend'} / container_spec_cpu_period{namespace=~'$namespace',container=~'query-frontend'})", + "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'query-frontend', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1435,21 +1435,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'query-frontend'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'query-frontend'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'query-frontend', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1534,7 +1534,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/query-frontend'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1640,21 +1640,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'querier'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{container=~'querier'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'querier'} / container_spec_cpu_period{namespace=~'$namespace',container=~'querier'})", + "expr": "min(container_spec_cpu_quota{container=~'querier'} / container_spec_cpu_period{container=~'querier'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'querier', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{container=~'querier', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1748,21 +1748,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'querier'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{container=~'querier'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'querier'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{container=~'querier'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'querier', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{container=~'querier', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -1847,7 +1847,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/querier'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", @@ -1953,21 +1953,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{namespace=~'$namespace',container=~'compactor'}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{container=~'compactor'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{namespace=~'$namespace',container=~'compactor'} / container_spec_cpu_period{namespace=~'$namespace',container=~'compactor'})", + "expr": "min(container_spec_cpu_quota{container=~'compactor'} / container_spec_cpu_period{container=~'compactor'})", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'compactor', resource='cpu'} > 0)", + "expr": "min(kube_pod_container_resource_requests{container=~'compactor', resource='cpu'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -2061,21 +2061,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (container_memory_working_set_bytes{namespace=~'$namespace',container=~'compactor'})", + "expr": "sum by(pod) (container_memory_working_set_bytes{container=~'compactor'})", "format": "time_series", "interval": "1m", "legendFormat": "{{pod}}", "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{namespace=~'$namespace',container=~'compactor'} > 0)", + "expr": "min(container_spec_memory_limit_bytes{container=~'compactor'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{namespace=~'$namespace',container=~'compactor', resource='memory'} > 0)", + "expr": "min(kube_pod_container_resource_requests{container=~'compactor', resource='memory'} > 0)", "format": "time_series", "interval": "1m", "legendFormat": "request", @@ -2160,7 +2160,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{job=~'($namespace)/compactor'})", + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", "format": "time_series", "interval": "1m", "legendFormat": "{{instance}}", diff --git a/src/grafana_dashboards/tempo-rollout-progress.json b/src/grafana_dashboards/tempo-rollout-progress.json index fb627be..5a4dc54 100644 --- a/src/grafana_dashboards/tempo-rollout-progress.json +++ b/src/grafana_dashboards/tempo-rollout-progress.json @@ -108,7 +108,7 @@ "steppedLine": false, "targets": [ { - "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{namespace=~'$namespace',statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", + "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", "format": null, "intervalFactor": null, "legendFormat": "{{tempo_service}}", @@ -116,7 +116,7 @@ "step": null }, { - "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{namespace=~'$namespace',deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{namespace=~'$namespace'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", + "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", "format": null, "intervalFactor": null, "legendFormat": "{{tempo_service}}", @@ -230,7 +230,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -354,7 +354,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -474,7 +474,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -598,7 +598,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))\n", + "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))\n", "format": null, "instant": false, "interval": "", @@ -714,7 +714,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -838,7 +838,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -958,7 +958,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}[$__rate_interval]))\n", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*'}[$__rate_interval]))\n", "format": null, "instant": false, "interval": "", @@ -1082,7 +1082,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}))\n", + "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'tempo_api_.*'}))\n", "format": null, "instant": false, "interval": "", @@ -1212,7 +1212,7 @@ "steppedLine": false, "targets": [ { - "expr": "kube_deployment_status_replicas_unavailable{namespace=~'$namespace', deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", + "expr": "kube_deployment_status_replicas_unavailable{ deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", "format": null, "instant": true, "interval": "", @@ -1222,7 +1222,7 @@ "step": null }, { - "expr": "kube_statefulset_status_replicas_current{namespace=~'$namespace', statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'} -\nkube_statefulset_status_replicas_ready {namespace=~'$namespace', statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", + "expr": "kube_statefulset_status_replicas_current{ statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'} -\nkube_statefulset_status_replicas_ready { statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", "format": null, "instant": true, "interval": "", @@ -1299,7 +1299,7 @@ "id": 11, "targets": [ { - "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{namespace=~'$namespace',container=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'version', '$1', 'image', '.*:(.+)-.*'\n )\n)\n", + "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{container=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'version', '$1', 'image', '.*:(.+)-.*'\n )\n)\n", "instant": true, "legendFormat": "", "refId": "A" @@ -1381,14 +1381,14 @@ "steppedLine": false, "targets": [ { - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))[1h:])\n)\n", + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))[1h:])\n)\n", "format": "time_series", "interval": "1m", "legendFormat": "writes", "legendLink": null }, { - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route=~'tempo_api_.*'}))[1h:])\n)\n", + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'tempo_api_.*'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'tempo_api_.*'}))[1h:])\n)\n", "format": "time_series", "interval": "1m", "legendFormat": "reads", @@ -1505,7 +1505,7 @@ "options": [ ], - "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "query": "label_values(tempo_build_info{}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/src/grafana_dashboards/tempo-tenants.json b/src/grafana_dashboards/tempo-tenants.json index 99622c9..ed6f2f0 100644 --- a/src/grafana_dashboards/tempo-tenants.json +++ b/src/grafana_dashboards/tempo-tenants.json @@ -89,7 +89,7 @@ ], "targets": [ { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor'})\n) by (limit_name)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{})\n) by (limit_name)\n", "format": "table", "instant": true, "legendFormat": "", @@ -196,21 +196,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_distributor_bytes_received_total{job=~'($namespace)/distributor',tenant='$tenant'}[$__rate_interval]))", + "expr": "sum(rate(tempo_distributor_bytes_received_total{tenant='$tenant'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "received", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='ingestion_rate_limit_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='ingestion_rate_limit_bytes'})\n) by (ingestion_rate_limit_bytes)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_rate_limit_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_rate_limit_bytes'})\n) by (ingestion_rate_limit_bytes)\n", "format": "time_series", "interval": "1m", "legendFormat": "limit", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='ingestion_burst_size_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='ingestion_burst_size_bytes'})\n) by (ingestion_burst_size_bytes)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_burst_size_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_burst_size_bytes'})\n) by (ingestion_burst_size_bytes)\n", "format": "time_series", "interval": "1m", "legendFormat": "burst limit", @@ -295,14 +295,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_distributor_spans_received_total{job=~'($namespace)/distributor',tenant='$tenant'}[$__rate_interval]))", + "expr": "sum(rate(tempo_distributor_spans_received_total{tenant='$tenant'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "accepted", "legendLink": null }, { - "expr": "sum(rate(tempo_discarded_spans_total{job=~'($namespace)/distributor',tenant='$tenant'}[$__rate_interval])) by (reason)", + "expr": "sum(rate(tempo_discarded_spans_total{tenant='$tenant'}[$__rate_interval])) by (reason)", "format": "time_series", "interval": "1m", "legendFormat": "refused {{ reason }}", @@ -396,21 +396,21 @@ "steppedLine": false, "targets": [ { - "expr": "max(tempo_ingester_live_traces{job=~'($namespace)/ingester',tenant='$tenant'})", + "expr": "max(tempo_ingester_live_traces{tenant='$tenant'})", "format": "time_series", "interval": "1m", "legendFormat": "live traces", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='max_global_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='max_global_traces_per_user'})\n) by (max_global_traces_per_user)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_global_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_global_traces_per_user'})\n) by (max_global_traces_per_user)\n", "format": "time_series", "interval": "1m", "legendFormat": "global limit", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='max_local_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='max_local_traces_per_user'})\n) by (max_local_traces_per_user)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_local_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_local_traces_per_user'})\n) by (max_local_traces_per_user)\n", "format": "time_series", "interval": "1m", "legendFormat": "local limit", @@ -507,7 +507,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_query_frontend_queries_total{job=~'($namespace)/query-frontend',tenant='$tenant',op='traces'}[$__rate_interval])) by (status)", + "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='traces'}[$__rate_interval])) by (status)", "format": "time_series", "interval": "1m", "legendFormat": "{{ status }}", @@ -592,7 +592,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_query_frontend_queries_total{job=~'($namespace)/query-frontend',tenant='$tenant',op='search'}[$__rate_interval])) by (status)", + "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='search'}[$__rate_interval])) by (status)", "format": "time_series", "interval": "1m", "legendFormat": "{{ status }}", @@ -683,7 +683,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tempodb_blocklist_length{job=~'($namespace)/compactor',tenant='$tenant'})", + "expr": "avg(tempodb_blocklist_length{tenant='$tenant'})", "format": "time_series", "interval": "1m", "legendFormat": "length", @@ -762,7 +762,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tempodb_compaction_outstanding_blocks{job=~'($namespace)/compactor',tenant='$tenant'})\n/\ncount(tempo_build_info{job=~'($namespace)/compactor'})\n", + "expr": "sum(tempodb_compaction_outstanding_blocks{tenant='$tenant'})\n/\ncount(tempo_build_info{})\n", "format": "time_series", "interval": "1m", "legendFormat": "blocks", @@ -853,7 +853,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{job=~'($namespace)/metrics-generator',tenant='$tenant'}[$__rate_interval]))", + "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{tenant='$tenant'}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "rate", @@ -942,14 +942,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tempo_metrics_generator_registry_active_series{job=~'($namespace)/metrics-generator',tenant='$tenant'})", + "expr": "sum(tempo_metrics_generator_registry_active_series{tenant='$tenant'})", "format": "time_series", "interval": "1m", "legendFormat": "{{ tenant }}", "legendLink": null }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{job=~'($namespace)/compactor',user='$tenant',limit_name='metrics_generator_max_active_series'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{job=~'($namespace)/compactor',limit_name='metrics_generator_max_active_series'})\n) by (metrics_generator_max_active_series)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='metrics_generator_max_active_series'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='metrics_generator_max_active_series'})\n) by (metrics_generator_max_active_series)\n", "format": "time_series", "interval": "1m", "legendFormat": "limit", @@ -1072,7 +1072,7 @@ "options": [ ], - "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "query": "label_values(tempo_build_info{}, namespace)", "refresh": 1, "regex": "", "sort": 2, @@ -1099,7 +1099,7 @@ "options": [ ], - "query": "label_values(tempodb_blocklist_length{job=~'($namespace)/compactor'}, tenant)", + "query": "label_values(tempodb_blocklist_length{}, tenant)", "refresh": 1, "regex": "", "sort": 2, diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json index e5056ab..abda572 100644 --- a/src/grafana_dashboards/tempo-writes.json +++ b/src/grafana_dashboards/tempo-writes.json @@ -74,7 +74,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -159,7 +159,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -168,7 +168,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -177,7 +177,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/cortex-gw(-internal)?', route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -276,7 +276,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {job=~'($namespace)/cortex-gw(-internal)?', __name__=~'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+'},\n 'grpc_status', '$1', '__name__', 'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)'\n )\n [$__interval:$__interval]\n )\n)\n", + "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {__name__=~'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+'},\n 'grpc_status', '$1', '__name__', 'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)'\n )\n [$__interval:$__interval]\n )\n)\n", "format": "time_series", "interval": "1m", "legendFormat": "{{grpc_status}}", @@ -397,14 +397,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_receiver_accepted_spans{job=~'($namespace)/distributor'}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "accepted", "legendLink": null }, { - "expr": "sum(rate(tempo_receiver_refused_spans{job=~'($namespace)/distributor'}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", "format": "time_series", "interval": "1m", "legendFormat": "refused", @@ -489,7 +489,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{job=~'($namespace)/distributor'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -498,7 +498,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{job=~'($namespace)/distributor'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -507,7 +507,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{job=~'($namespace)/distributor'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{job=~'($namespace)/distributor'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -614,7 +614,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester', route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -699,7 +699,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -708,7 +708,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -717,7 +717,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_request_duration_seconds_sum{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{job=~'($namespace)/ingester',route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -824,7 +824,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -909,7 +909,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -918,7 +918,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -927,7 +927,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/ingester',method='Memcache.Put'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1034,7 +1034,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1119,7 +1119,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1128,7 +1128,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1137,7 +1137,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/ingester',operation=~'(PUT|POST)'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1244,7 +1244,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1329,7 +1329,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1338,7 +1338,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1347,7 +1347,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{job=~'($namespace)/compactor',method='Memcache.Put'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1454,7 +1454,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1539,7 +1539,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1548,7 +1548,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1557,7 +1557,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{job=~'($namespace)/compactor',operation=~'(PUT|POST)'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1682,7 +1682,7 @@ "options": [ ], - "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "query": "label_values(tempo_build_info{}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index d7cf799..a02cc7a 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -3700,7 +3700,7 @@ }, "timezone": "", "title": "Tempo Overview", - "uid": "a6175b9cc7ec20591890117c39580030", + "uid": "a6175b9cc7ec20591890117c39580049", "version": 1, "weekStart": "" } \ No newline at end of file From 7799ebf602851f4a874e87fa1e1e2835aabf9054 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Mon, 19 Aug 2024 16:58:53 +0200 Subject: [PATCH 05/14] Remove panels that aren't useful in our Tempo architecture --- src/grafana_dashboards/tempo-operational.json | 2199 +++-------------- 1 file changed, 332 insertions(+), 1867 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index 7e0e8ba..f7fec73 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -743,101 +743,6 @@ "title": "data volume free", "type": "timeseries" }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 46, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "rate(promtail_custom_bad_words_total{app=~'$component.*'}[$__rate_interval])", - "interval": "", - "legendFormat": "{{exported_pod}}", - "refId": "A" - } - ], - "title": "bad words", - "type": "timeseries" - }, { "collapsed": true, "datasource": { @@ -1640,9 +1545,9 @@ "h": 5, "w": 4, "x": 0, - "y": 8 + "y": 13 }, - "id": 34, + "id": 90, "options": { "legend": { "calcs": [ @@ -1667,7 +1572,7 @@ "refId": "A" } ], - "title": "Queries/Sec (cortex-gw)", + "title": "Queries/Sec (Query Frontend)", "type": "timeseries" }, { @@ -1735,9 +1640,9 @@ "h": 5, "w": 6, "x": 4, - "y": 8 + "y": 13 }, - "id": 78, + "id": 17, "options": { "legend": { "calcs": [ @@ -1773,7 +1678,7 @@ "refId": "C" } ], - "title": "Query Latency (Gateway)", + "title": "Query Latency (Query Frontend)", "type": "timeseries" }, { @@ -1841,9 +1746,9 @@ "h": 5, "w": 4, "x": 12, - "y": 8 + "y": 13 }, - "id": 97, + "id": 98, "options": { "legend": { "calcs": [ @@ -1870,7 +1775,7 @@ "stepMode": "min" } ], - "title": "Search Queries/Sec (cortex-gw)", + "title": "Search Queries/Sec (Query Frontend)", "type": "timeseries" }, { @@ -1938,9 +1843,9 @@ "h": 5, "w": 6, "x": 16, - "y": 8 + "y": 13 }, - "id": 93, + "id": 94, "options": { "legend": { "calcs": [ @@ -1968,7 +1873,6 @@ { "exemplar": true, "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", - "hide": false, "interval": "", "legendFormat": ".9", "refId": "B", @@ -1977,14 +1881,13 @@ { "exemplar": true, "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", - "hide": false, "interval": "", "legendFormat": ".5", "refId": "C", "stepMode": "min" } ], - "title": "Search Query Latency (Gateway)", + "title": "Search Query Latency (Query Frontend)", "type": "timeseries" }, { @@ -2052,9 +1955,9 @@ "h": 5, "w": 4, "x": 0, - "y": 13 + "y": 18 }, - "id": 90, + "id": 91, "options": { "legend": { "calcs": [ @@ -2072,14 +1975,14 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", "refId": "A" } ], - "title": "Queries/Sec (Query Frontend)", + "title": "Queries/Sec (Querier)", "type": "timeseries" }, { @@ -2147,9 +2050,9 @@ "h": 5, "w": 6, "x": 4, - "y": 13 + "y": 18 }, - "id": 17, + "id": 89, "options": { "legend": { "calcs": [ @@ -2167,25 +2070,25 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" } ], - "title": "Query Latency (Query Frontend)", + "title": "Query Latency (Querier)", "type": "timeseries" }, { @@ -2253,9 +2156,9 @@ "h": 5, "w": 4, "x": 12, - "y": 13 + "y": 18 }, - "id": 98, + "id": 99, "options": { "legend": { "calcs": [ @@ -2274,7 +2177,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2282,7 +2185,7 @@ "stepMode": "min" } ], - "title": "Search Queries/Sec (Query Frontend)", + "title": "Search Queries/Sec (Querier)", "type": "timeseries" }, { @@ -2350,9 +2253,9 @@ "h": 5, "w": 6, "x": 16, - "y": 13 + "y": 18 }, - "id": 94, + "id": 95, "options": { "legend": { "calcs": [ @@ -2371,7 +2274,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2379,7 +2282,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2387,14 +2290,14 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", "stepMode": "min" } ], - "title": "Search Query Latency (Query Frontend)", + "title": "Search Query Latency (Querier)", "type": "timeseries" }, { @@ -2462,9 +2365,9 @@ "h": 5, "w": 4, "x": 0, - "y": 18 + "y": 23 }, - "id": 91, + "id": 92, "options": { "legend": { "calcs": [ @@ -2482,14 +2385,14 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", "refId": "A" } ], - "title": "Queries/Sec (Querier)", + "title": "Queries/Sec (Ingester)", "type": "timeseries" }, { @@ -2557,9 +2460,9 @@ "h": 5, "w": 6, "x": 4, - "y": 18 + "y": 23 }, - "id": 89, + "id": 3, "options": { "legend": { "calcs": [ @@ -2577,25 +2480,25 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" } ], - "title": "Query Latency (Querier)", + "title": "Query Latency (Ingester)", "type": "timeseries" }, { @@ -2663,9 +2566,9 @@ "h": 5, "w": 4, "x": 12, - "y": 18 + "y": 23 }, - "id": 99, + "id": 100, "options": { "legend": { "calcs": [ @@ -2684,7 +2587,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2692,7 +2595,7 @@ "stepMode": "min" } ], - "title": "Search Queries/Sec (Querier)", + "title": "Search Queries/Sec (Ingester)", "type": "timeseries" }, { @@ -2760,9 +2663,9 @@ "h": 5, "w": 6, "x": 16, - "y": 18 + "y": 23 }, - "id": 95, + "id": 96, "options": { "legend": { "calcs": [ @@ -2781,7 +2684,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2789,7 +2692,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2797,18 +2700,19 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C", "stepMode": "min" } ], - "title": "Search Query Latency (Querier)", + "title": "Search Query Latency (Ingester)", "type": "timeseries" }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -2871,10 +2775,10 @@ "gridPos": { "h": 5, "w": 4, - "x": 0, - "y": 23 + "x": 12, + "y": 28 }, - "id": 92, + "id": 110, "options": { "legend": { "calcs": [ @@ -2892,18 +2796,27 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (status_code)", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", - "refId": "A" + "range": true, + "refId": "A", + "stepMode": "min" } ], - "title": "Queries/Sec (Ingester)", + "title": "Metrics Summary QPS (Query Frontend)", "type": "timeseries" }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -2966,10 +2879,10 @@ "gridPos": { "h": 5, "w": 6, - "x": 4, - "y": 23 + "x": 16, + "y": 28 }, - "id": 3, + "id": 111, "options": { "legend": { "calcs": [ @@ -2987,29 +2900,54 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", - "refId": "A" + "range": true, + "refId": "A", + "stepMode": "min" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", - "refId": "B" + "range": true, + "refId": "B", + "stepMode": "min" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", - "refId": "C" + "range": true, + "refId": "C", + "stepMode": "min" } ], - "title": "Query Latency (Ingester)", + "title": "Metrics Summary Latency (Query Frontend)", "type": "timeseries" }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -3072,10 +3010,10 @@ "gridPos": { "h": 5, "w": 4, - "x": 12, - "y": 23 + "x": 0, + "y": 33 }, - "id": 100, + "id": 112, "options": { "legend": { "calcs": [ @@ -3093,20 +3031,25 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (status_code)", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A", - "stepMode": "min" + "legendFormat": "{{status_code}} {{route}}", + "range": true, + "refId": "A" } ], - "title": "Search Queries/Sec (Ingester)", + "title": "Metrics Summary QPS (Querier)", "type": "timeseries" }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -3169,10 +3112,10 @@ "gridPos": { "h": 5, "w": 6, - "x": 16, - "y": 23 + "x": 4, + "y": 33 }, - "id": 96, + "id": 113, "options": { "legend": { "calcs": [ @@ -3190,31 +3133,43 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", - "refId": "A", - "stepMode": "min" + "range": true, + "refId": "A" }, { - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", - "refId": "B", - "stepMode": "min" + "range": true, + "refId": "B" }, { - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", - "refId": "C", - "stepMode": "min" + "range": true, + "refId": "C" } ], - "title": "Search Query Latency (Ingester)", + "title": "Metrics Summary Latency (Querier)", "type": "timeseries" }, { @@ -3282,10 +3237,10 @@ "gridPos": { "h": 5, "w": 4, - "x": 0, - "y": 28 + "x": 12, + "y": 33 }, - "id": 102, + "id": 106, "options": { "legend": { "calcs": [ @@ -3308,7 +3263,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics_summary'}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -3316,7 +3271,7 @@ "refId": "A" } ], - "title": "Metrics Summary QPS (cortex-gw)", + "title": "GetMetrics QPS (Metrics Generator)", "type": "timeseries" }, { @@ -3384,10 +3339,10 @@ "gridPos": { "h": 5, "w": 6, - "x": 4, - "y": 28 + "x": 16, + "y": 33 }, - "id": 103, + "id": 107, "options": { "legend": { "calcs": [ @@ -3410,7 +3365,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3422,7 +3377,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3434,16 +3389,34 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics_summary'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, "refId": "C" } ], - "title": "Metrics Summary Latency (Gateway)", + "title": "GetMetrics Latency (Metrics Generator)", "type": "timeseries" - }, + } + ], + "title": "Read", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [ { "datasource": { "type": "prometheus", @@ -3461,7 +3434,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3476,10 +3449,10 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -3492,7 +3465,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3508,11 +3482,11 @@ }, "gridPos": { "h": 5, - "w": 4, - "x": 12, - "y": 28 + "w": 3, + "x": 0, + "y": 4 }, - "id": 110, + "id": 10, "options": { "legend": { "calcs": [ @@ -3520,14 +3494,14 @@ ], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { "datasource": { @@ -3535,17 +3509,14 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics.*'}[$__rate_interval])) by (status_code)", - "hide": false, + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", "interval": "", - "legendFormat": "{{status_code}}", + "legendFormat": "{{pod}}", "range": true, - "refId": "A", - "stepMode": "min" + "refId": "A" } ], - "title": "Metrics Summary QPS (Query Frontend)", + "title": "Spans Received", "type": "timeseries" }, { @@ -3580,1209 +3551,10 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 28 - }, - "id": 111, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A", - "stepMode": "min" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B", - "stepMode": "min" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C", - "stepMode": "min" - } - ], - "title": "Metrics Summary Latency (Query Frontend)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 33 - }, - "id": 112, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}} {{route}}", - "range": true, - "refId": "A" - } - ], - "title": "Metrics Summary QPS (Querier)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 4, - "y": 33 - }, - "id": 113, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" - } - ], - "title": "Metrics Summary Latency (Querier)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 33 - }, - "id": 106, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "range": true, - "refId": "A" - } - ], - "title": "GetMetrics QPS (Metrics Generator)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 33 - }, - "id": 107, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" - } - ], - "title": "GetMetrics Latency (Metrics Generator)", - "type": "timeseries" - } - ], - "title": "Read", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 19, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 4 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Spans Received", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 4 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Traces Created", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 4 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{job='$namespace/ingester'}[1h]))", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Blocks Flushed (1h)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 4 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Failed Flushes", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 4 - }, - "id": 26, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Blocks Cleared (1h)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 4 - }, - "id": 36, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Flush Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -4813,18 +3585,18 @@ "gridPos": { "h": 5, "w": 3, - "x": 18, + "x": 3, "y": 4 }, - "id": 114, + "id": 9, "options": { "legend": { "calcs": [ - "lastNotNull" + ], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -4838,15 +3610,15 @@ "type": "prometheus", "uid": "${prometheusds}" }, - "editorMode": "code", - "expr": "sum(tempo_distributor_queue_length{cluster=~'$cluster'}) by (name)", + "editorMode": "builder", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", "interval": "", - "legendFormat": "{{name}}", + "legendFormat": "", "range": true, "refId": "A" } ], - "title": "Distributor Queue Length", + "title": "Traces Created", "type": "timeseries" }, { @@ -4912,12 +3684,12 @@ ] }, "gridPos": { - "h": 10, - "w": 7, - "x": 0, - "y": 9 + "h": 5, + "w": 3, + "x": 6, + "y": 4 }, - "id": 71, + "id": 8, "options": { "legend": { "calcs": [ @@ -4925,7 +3697,7 @@ ], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -4935,13 +3707,13 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (status_code)", + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{}[1h]))", "interval": "", - "legendFormat": "{{status_code}}", + "legendFormat": "{{pod}}", "refId": "A" } ], - "title": "Pushes/sec (gateway)", + "title": "Blocks Flushed (1h)", "type": "timeseries" }, { @@ -5000,19 +3772,19 @@ } ] }, - "unit": "none" + "unit": "short" }, "overrides": [ ] }, "gridPos": { - "h": 10, - "w": 6, - "x": 7, - "y": 9 + "h": 5, + "w": 3, + "x": 9, + "y": 4 }, - "id": 72, + "id": 12, "options": { "legend": { "calcs": [ @@ -5020,7 +3792,7 @@ ], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -5030,19 +3802,13 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", + "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", "interval": "", - "legendFormat": "accepted", + "legendFormat": "{{pod}}", "refId": "A" - }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", - "interval": "", - "legendFormat": "refused", - "refId": "B" } ], - "title": "OTEL Distributor Spans/second", + "title": "Failed Flushes", "type": "timeseries" }, { @@ -5101,7 +3867,7 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [ @@ -5109,11 +3875,11 @@ }, "gridPos": { "h": 5, - "w": 7, - "x": 13, - "y": 9 + "w": 3, + "x": 12, + "y": 4 }, - "id": 79, + "id": 26, "options": { "legend": { "calcs": [ @@ -5121,7 +3887,7 @@ ], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -5131,25 +3897,13 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", "interval": "", - "legendFormat": ".99", + "legendFormat": "", "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" } ], - "title": "Push Latency (Gateway)", + "title": "Blocks Cleared (1h)", "type": "timeseries" }, { @@ -5167,7 +3921,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { @@ -5178,11 +3932,11 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "always", "spanNulls": false, "stacking": { "group": "A", @@ -5216,11 +3970,11 @@ }, "gridPos": { "h": 5, - "w": 7, - "x": 13, - "y": 14 + "w": 3, + "x": 15, + "y": 4 }, - "id": 2, + "id": 36, "options": { "legend": { "calcs": [ @@ -5238,30 +3992,29 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" } ], - "title": "Push Latency (Ingester)", + "title": "Flush Duration", "type": "timeseries" }, { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -5323,12 +4076,12 @@ ] }, "gridPos": { - "h": 5, + "h": 10, "w": 6, - "x": 7, - "y": 19 + "x": 0, + "y": 9 }, - "id": 109, + "id": 72, "options": { "legend": { "calcs": [ @@ -5346,36 +4099,23 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", "interval": "", "legendFormat": "accepted", - "range": true, "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", "interval": "", - "legendFormat": "refused {{reason}}", - "range": true, + "legendFormat": "refused", "refId": "B" } ], - "title": "Generator Spans/second", + "title": "OTEL Distributor Spans/second", "type": "timeseries" }, { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -5439,10 +4179,10 @@ "gridPos": { "h": 5, "w": 7, - "x": 13, - "y": 19 + "x": 6, + "y": 14 }, - "id": 108, + "id": 2, "options": { "legend": { "calcs": [ @@ -5460,65 +4200,30 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", - "range": true, "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", - "range": true, "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", - "range": true, "refId": "C" } ], - "title": "Push Latency (Generator)", + "title": "Push Latency (Ingester)", "type": "timeseries" - } - ], - "title": "Write", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 74, - "panels": [ + }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -5527,6 +4232,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -5538,6 +4245,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5545,7 +4253,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -5554,9 +4262,6 @@ "mode": "off" } }, - "links": [ - - ], "mappings": [ ], @@ -5564,7 +4269,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5572,19 +4278,19 @@ } ] }, - "unit": "short" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, + "h": 5, + "w": 6, "x": 0, - "y": 15 + "y": 19 }, - "id": 75, + "id": 109, "options": { "legend": { "calcs": [ @@ -5599,20 +4305,39 @@ "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_count{}[$__rate_interval])) by (status_code, method)", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", "interval": "", - "legendFormat": "{{status_code}}-{{method}}", + "legendFormat": "accepted", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", + "interval": "", + "legendFormat": "refused {{reason}}", + "range": true, + "refId": "B" } ], - "title": "Requests/Second", + "title": "Generator Spans/second", "type": "timeseries" }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -5621,6 +4346,8 @@ "mode": "palette-classic" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -5632,6 +4359,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5639,7 +4367,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -5648,9 +4376,6 @@ "mode": "off" } }, - "links": [ - - ], "mappings": [ ], @@ -5658,7 +4383,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5673,12 +4399,12 @@ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 + "h": 5, + "w": 7, + "x": 6, + "y": 19 }, - "id": 76, + "id": 108, "options": { "legend": { "calcs": [ @@ -5693,32 +4419,50 @@ "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{}[$__rate_interval])) by (method, le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", - "legendFormat": ".99-{{method}}", + "legendFormat": ".99", + "range": true, "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{}[$__rate_interval])) by (method, le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", - "legendFormat": ".9-{{method}}", + "legendFormat": ".9", + "range": true, "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{}[$__rate_interval])) by (method, le))", + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", "interval": "", - "legendFormat": ".5-{{method}}", + "legendFormat": ".5", + "range": true, "refId": "C" } ], - "title": "Latency By Operation", + "title": "Push Latency (Generator)", "type": "timeseries" } ], - "title": "Memcached", + "title": "Write", "type": "row" }, { @@ -5952,7 +4696,6 @@ "panels": [ { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -6018,104 +4761,6 @@ "x": 0, "y": 17 }, - "id": 58, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "expr": "gauge_memberlist_health_score{}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Gossip Health", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 17 - }, "id": 62, "options": { "legend": { @@ -6207,7 +4852,7 @@ "gridPos": { "h": 8, "w": 6, - "x": 12, + "x": 6, "y": 17 }, "id": 63, @@ -6308,7 +4953,7 @@ "gridPos": { "h": 8, "w": 6, - "x": 18, + "x": 12, "y": 17 }, "id": 60, @@ -6356,176 +5001,6 @@ "title": "Ring/Memberlist", "type": "row" }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 69, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 77, - "options": { - "graph": { - - }, - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.0-14141pre", - "targets": [ - { - "expr": "sum(rate(tempo_vulture_trace_error_total{}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{error}}", - "refId": "A" - } - ], - "title": "Vulture Query Errors", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 67, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": { - - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempo_vulture_trace_error_total{}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{secondsago}}", - "refId": "A" - } - ], - "title": "Average Vulture Query Errors", - "type": "bargauge" - } - ], - "title": "Vulture", - "type": "row" - }, { "collapsed": true, "datasource": { @@ -7071,19 +5546,9 @@ "selected": false, "text": "querier", "value": "querier" - }, - { - "selected": false, - "text": "cortex-gw", - "value": "cortex-gw" - }, - { - "selected": false, - "text": "cortex-gw-internal", - "value": "cortex-gw-internal" } ], - "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier,cortex-gw,cortex-gw-internal", + "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier", "queryValue": "", "skipUrlSync": false, "type": "custom" From 7158cdd480e9659583e3d470211ea26521134afe Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Mon, 19 Aug 2024 17:50:10 +0200 Subject: [PATCH 06/14] Reads dashboard - removed unused components --- src/grafana_dashboards/tempo-reads.json | 420 ------------------------ 1 file changed, 420 deletions(-) diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json index 7d79f12..fa74780 100644 --- a/src/grafana_dashboards/tempo-reads.json +++ b/src/grafana_dashboards/tempo-reads.json @@ -24,216 +24,6 @@ ], "refresh": "", "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval])) by (route)", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, { "collapse": false, "height": "250px", @@ -1074,216 +864,6 @@ "title": "Ingester", "titleSize": "h6" }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{method=~'Memcache.Get|Memcache.GetMulti'}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memcached", - "titleSize": "h6" - }, { "collapse": false, "height": "250px", From 31d956e9af7b96e6bc00c04a6be47fa049d37c54 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Tue, 20 Aug 2024 10:33:36 +0200 Subject: [PATCH 07/14] Remove resources dashboard that doesn't do well with our architecture --- src/grafana_dashboards/tempo-resources.json | 2333 ------------------- 1 file changed, 2333 deletions(-) delete mode 100644 src/grafana_dashboards/tempo-resources.json diff --git a/src/grafana_dashboards/tempo-resources.json b/src/grafana_dashboards/tempo-resources.json deleted file mode 100644 index 0ceceed..0000000 --- a/src/grafana_dashboards/tempo-resources.json +++ /dev/null @@ -1,2333 +0,0 @@ -{ - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo Dashboards", - "type": "dashboards" - } - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Distributor", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Metrics-generator", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{} / container_spec_cpu_period{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Query Frontend", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{container=~'querier'}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{container=~'querier'} / container_spec_cpu_period{container=~'querier'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{container=~'querier', resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{container=~'querier'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{container=~'querier'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{container=~'querier', resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Querier", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{container=~'compactor'}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{container=~'compactor'} / container_spec_cpu_period{container=~'compactor'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{container=~'compactor', resource='cpu'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - }, - { - "alias": "request", - "color": "#FCE300", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (container_memory_working_set_bytes{container=~'compactor'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{container=~'compactor'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{container=~'compactor', resource='memory'} > 0)", - "format": "time_series", - "interval": "1m", - "legendFormat": "request", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Compactor", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "cluster", - "multi": true, - "name": "cluster", - "options": [ - - ], - "query": "label_values(tempo_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": true, - "name": "namespace", - "options": [ - - ], - "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Resources", - "uid": "", - "version": 0 -} \ No newline at end of file From e5c334d1725f86eb7d5344000949b614d0ae09b5 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Tue, 20 Aug 2024 11:03:07 +0200 Subject: [PATCH 08/14] Add links to main tempo dashboard and remove wrong sections from writes --- src/grafana_dashboards/tempo-writes.json | 805 +---------------------- src/grafana_dashboards/tempo.json | 15 +- 2 files changed, 41 insertions(+), 779 deletions(-) diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json index abda572..d510ffc 100644 --- a/src/grafana_dashboards/tempo-writes.json +++ b/src/grafana_dashboards/tempo-writes.json @@ -28,639 +28,6 @@ "collapse": false, "height": "250px", "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route='opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {__name__=~'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+'},\n 'grpc_status', '$1', '__name__', 'envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)'\n )\n [$__interval:$__interval]\n )\n)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{grpc_status}}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - - } - }, - "overrides": [ - - ] - }, - "id": 4, - "options": { - "content": "Visit [Status codes and their use in gRPC](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md)\n\nCode | Number | Description\n---|---|---\nOK | 0 | Not an error; returned on success.\nCANCELLED | 1 | The operation was cancelled, typically by the caller.\nUNKNOWN | 2 | Unknown error. For example, this error may be returned when a Status value received from another address space belongs to an error space that is not known in this address space. Also errors raised by APIs that do not return enough error information may be converted to this error.\nINVALID_ARGUMENT | 3 | The client specified an invalid argument. Note that this differs from FAILED_PRECONDITION. INVALID_ARGUMENT indicates arguments that are problematic regardless of the state of the system (e.g., a malformed file name).\nDEADLINE_EXCEEDED | 4 | The deadline expired before the operation could complete. For operations that change the state of the system, this error may be returned even if the operation has completed successfully. For example, a successful response from a server could have been delayed long\nNOT_FOUND | 5 | Some requested entity (e.g., file or directory) was not found. Note to server developers: if a request is denied for an entire class of users, such as gradual feature rollout or undocumented allowlist, NOT_FOUND may be used. If a request is denied for some users within a class of users, such as user-based access control, PERMISSION_DENIED must be used.\nALREADY_EXISTS | 6 | The entity that a client attempted to create (e.g., file or directory) already exists.\nPERMISSION_DENIED | 7 | The caller does not have permission to execute the specified operation. PERMISSION_DENIED must not be used for rejections caused by exhausting some resource (use RESOURCE_EXHAUSTED instead for those errors). PERMISSION_DENIED must not be used if the caller can not be identified (use UNAUTHENTICATED instead for those errors). This error code does not imply the request is valid or the requested entity exists or satisfies other pre-conditions.\nRESOURCE_EXHAUSTED | 8 | Some resource has been exhausted, perhaps a per-user quota, or perhaps the entire file system is out of space.\nFAILED_PRECONDITION | 9 | The operation was rejected because the system is not in a state required for the operation's execution. For example, the directory to be deleted is non-empty, an rmdir operation is applied to a non-directory, etc. Service implementors can use the following guidelines to decide between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE: (a) Use UNAVAILABLE if the client can retry just the failing call. (b) Use ABORTED if the client should retry at a higher level (e.g., when a client-specified test-and-set fails, indicating the client should restart a read-modify-write sequence). (c) Use FAILED_PRECONDITION if the client should not retry until the system state has been explicitly fixed. E.g., if an 'rmdir' fails because the directory is non-empty, FAILED_PRECONDITION should be returned since the client should not retry unless the files are deleted from the directory.\nABORTED | 10 | The operation was aborted, typically due to a concurrency issue such as a sequencer check failure or transaction abort. See the guidelines above for deciding between FAILED_PRECONDITION, ABORTED, and UNAVAILABLE.\nOUT_OF_RANGE | 11 | The operation was attempted past the valid range. E.g., seeking or reading past end-of-file. Unlike INVALID_ARGUMENT, this error indicates a problem that may be fixed if the system state changes. For example, a 32-bit file system will generate INVALID_ARGUMENT if asked to read at an offset that is not in the range [0,2^32-1], but it will generate OUT_OF_RANGE if asked to read from an offset past the current file size. There is a fair bit of overlap between FAILED_PRECONDITION and OUT_OF_RANGE. We recommend using OUT_OF_RANGE (the more specific error) when it applies so that callers who are iterating through a space can easily look for an OUT_OF_RANGE error to detect when they are done.\nUNIMPLEMENTED | 12 | The operation is not implemented or is not supported/enabled in this service.\nINTERNAL | 13 | Internal errors. This means that some invariants expected by the underlying system have been broken. This error code is reserved for serious errors.\nUNAVAILABLE | 14 | The service is currently unavailable. This is most likely a transient condition, which can be corrected by retrying with a backoff. Note that it is not always safe to retry non-idempotent operations.\nDATA_LOSS | 15 | Unrecoverable data loss or corruption.\nUNAUTHENTICATED | 16 | The request does not have valid authentication credentials for the operation.\n", - "mode": "markdown" - }, - "span": 6, - "timeFrom": null, - "timeShift": null, - "title": "gRPC status codes", - "transparent": false, - "type": "text" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Envoy Proxy", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "accepted", - "legendLink": null - }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "refused", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Spans/Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Distributor", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, { "aliasColors": { @@ -670,7 +37,7 @@ "dashes": false, "datasource": "${prometheusds}", "fill": 1, - "id": 8, + "id": 5, "legend": { "avg": false, "current": false, @@ -699,136 +66,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "legendFormat": "accepted", + "legendLink": null }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" + "legendFormat": "refused", + "legendLink": null } ], "thresholds": [ @@ -836,7 +85,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "QPS", + "title": "Spans/Second", "tooltip": { "shared": true, "sort": 2, @@ -880,7 +129,7 @@ "dashes": false, "datasource": "${prometheusds}", "fill": 1, - "id": 10, + "id": 6, "legend": { "avg": false, "current": false, @@ -909,7 +158,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -918,7 +167,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -927,7 +176,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -981,7 +230,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Memcached - Ingester", + "title": "Distributor", "titleSize": "h6" }, { @@ -1005,7 +254,7 @@ "dashes": false, "datasource": "${prometheusds}", "fill": 10, - "id": 11, + "id": 7, "legend": { "avg": false, "current": false, @@ -1034,7 +283,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1090,7 +339,7 @@ "dashes": false, "datasource": "${prometheusds}", "fill": 1, - "id": 12, + "id": 8, "legend": { "avg": false, "current": false, @@ -1119,7 +368,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1128,7 +377,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1137,7 +386,7 @@ "step": 10 }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1191,7 +440,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Backend - Ingester", + "title": "Ingester", "titleSize": "h6" }, { @@ -1215,7 +464,7 @@ "dashes": false, "datasource": "${prometheusds}", "fill": 10, - "id": 13, + "id": 11, "legend": { "avg": false, "current": false, @@ -1244,7 +493,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", "format": "time_series", "interval": "1m", "legendFormat": "{{status}}", @@ -1300,7 +549,7 @@ "dashes": false, "datasource": "${prometheusds}", "fill": 1, - "id": 14, + "id": 12, "legend": { "avg": false, "current": false, @@ -1329,7 +578,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1338,7 +587,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{method='Memcache.Put'}[$__rate_interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1347,7 +596,7 @@ "step": 10 }, { - "expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{method='Memcache.Put'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{method='Memcache.Put'}[$__rate_interval])) by ()", + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1401,7 +650,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Memcached - Compactor", + "title": "Backend - Ingester", "titleSize": "h6" }, { diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index a02cc7a..15147aa 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -36,7 +36,20 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "links": [], + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], "liveNow": false, "panels": [ { From dfe619d847c8e5145d0d01e3c3ade6704ccc8060 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Tue, 20 Aug 2024 15:19:04 +0200 Subject: [PATCH 09/14] sum up spans + remove rollout-progress dashboard --- .../tempo-rollout-progress.json | 1555 ----------------- src/grafana_dashboards/tempo.json | 4 +- 2 files changed, 2 insertions(+), 1557 deletions(-) delete mode 100644 src/grafana_dashboards/tempo-rollout-progress.json diff --git a/src/grafana_dashboards/tempo-rollout-progress.json b/src/grafana_dashboards/tempo-rollout-progress.json deleted file mode 100644 index 5a4dc54..0000000 --- a/src/grafana_dashboards/tempo-rollout-progress.json +++ /dev/null @@ -1,1555 +0,0 @@ -{ - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo Dashboards", - "type": "dashboards" - } - ], - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - }, - { - "color": "yellow", - "value": 0.999 - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "percentunit" - } - }, - "fill": 1, - "gridPos": { - "h": 8, - "w": 10, - "x": 0, - "y": 0 - }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_statefulset_replicas{},\n 'tempo_service', '$1', 'statefulset', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", - "format": null, - "intervalFactor": null, - "legendFormat": "{{tempo_service}}", - "legendLink": null, - "step": null - }, - { - "expr": "(\n sum by(tempo_service) (\n label_replace(\n kube_deployment_status_replicas_updated{deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n /\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n) and (\n sum by(tempo_service) (\n label_replace(\n kube_deployment_spec_replicas{},\n 'tempo_service', '$1', 'deployment', '(.*?)(?:-zone-[a-z])?'\n )\n )\n > 0\n)\n", - "format": null, - "intervalFactor": null, - "legendFormat": "{{tempo_service}}", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Rollout progress", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "bargauge", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 2, - "x": 10, - "y": 0 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Writes - 2xx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.2 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 2, - "x": 12, - "y": 0 - }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Writes - 4xx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.01 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 2, - "x": 14, - "y": 0 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}[$__rate_interval]))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Writes - 5xx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.2 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Writes 99th latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 2, - "x": 10, - "y": 4 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*',status_code=~'2.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*'}[$__rate_interval]))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Reads - 2xx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.01 - }, - { - "color": "red", - "value": 0.05 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 2, - "x": 12, - "y": 4 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*',status_code=~'4.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*'}[$__rate_interval]))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Reads - 4xx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.01 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 2, - "x": 14, - "y": 4 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*',status_code=~'5.+'}[$__rate_interval])) /\nsum(rate(tempo_request_duration_seconds_count{route=~'tempo_api_.*'}[$__rate_interval]))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Reads - 5xx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "noValue": "", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 2.5 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 4 - }, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'tempo_api_.*'}))\n", - "format": null, - "instant": false, - "interval": "", - "intervalFactor": null, - "legendFormat": "", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Reads 99th latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "noValue": "All healthy", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "fill": 1, - "gridPos": { - "h": 8, - "w": 10, - "x": 0, - "y": 8 - }, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "options": { - "text": { - "titleSize": 14, - "valueSize": 14 - } - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kube_deployment_status_replicas_unavailable{ deployment=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", - "format": null, - "instant": true, - "interval": "", - "intervalFactor": null, - "legendFormat": "{{deployment}}", - "legendLink": null, - "step": null - }, - { - "expr": "kube_statefulset_status_replicas_current{ statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'} -\nkube_statefulset_status_replicas_ready { statefulset=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'}\n> 0\n", - "format": null, - "instant": true, - "interval": "", - "intervalFactor": null, - "legendFormat": "{{statefulset}}", - "legendLink": null, - "step": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Unhealthy pods", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "datasource": "${prometheusds}", - "fieldConfig": { - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "r.*" - }, - "properties": [ - { - "id": "custom.align", - "value": "center" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 10, - "y": 8 - }, - "id": 11, - "targets": [ - { - "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{container=~'.*(cortex-gw|distributor|ingester|query-frontend|querier|compactor|metrics-generator).*'},\n 'version', '$1', 'image', '.*:(.+)-.*'\n )\n)\n", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Pods count per version", - "transformations": [ - { - "id": "labelsToFields", - "options": { - "valueLabel": "version" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - } - } - }, - { - "id": "sortBy", - "options": { - "fields": { - - }, - "sort": [ - { - "field": "container" - } - ] - } - } - ], - "type": "table" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'opentelemetry_proto_collector_trace_v1_traceservice_export'}))[1h:])\n)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "writes", - "legendLink": null - }, - { - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'tempo_api_.*'} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (tempo_request_duration_seconds_bucket{route=~'tempo_api_.*'}))[1h:])\n)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "reads", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency vs 24h ago", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "refresh": "", - "rows": null, - "schemaVersion": 27, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": true, - "name": "cluster", - "options": [ - - ], - "query": "label_values(tempo_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": true, - "name": "namespace", - "options": [ - - ], - "query": "label_values(tempo_build_info{}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Rollout progress", - "uid": "217b16a8c5966b32c770225dea289b19", - "version": 0 -} \ No newline at end of file diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index 15147aa..9e2268a 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -760,7 +760,7 @@ "uid": "${prometheusds}" }, "editorMode": "builder", - "expr": "rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])", + "expr": "sum(rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "legendFormat": "accepted spans", "range": true, "refId": "A" @@ -771,7 +771,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])", + "expr": "sum(rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "hide": false, "legendFormat": "refused spans", "range": true, From 97ede448c1b604951b69817fa69e6a34240afbed Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 21 Aug 2024 09:15:06 +0200 Subject: [PATCH 10/14] Adjust dashboard UIDs to match our best practices --- src/grafana_dashboards/tempo-operational.json | 2 +- src/grafana_dashboards/tempo-reads.json | 2 +- src/grafana_dashboards/tempo-tenants.json | 2 +- src/grafana_dashboards/tempo-writes.json | 2 +- src/grafana_dashboards/tempo.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index f7fec73..a2494bc 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -5574,7 +5574,7 @@ }, "timezone": "", "title": "Tempo Operational", - "uid": "a6175b9cc7ec20591890117c39580030", + "uid": "tempo-coordinator-operational", "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json index fa74780..ae5aff4 100644 --- a/src/grafana_dashboards/tempo-reads.json +++ b/src/grafana_dashboards/tempo-reads.json @@ -1187,6 +1187,6 @@ }, "timezone": "utc", "title": "Tempo / Reads", - "uid": "", + "uid": "tempo-coordinator-reads", "version": 0 } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-tenants.json b/src/grafana_dashboards/tempo-tenants.json index ed6f2f0..7c544e6 100644 --- a/src/grafana_dashboards/tempo-tenants.json +++ b/src/grafana_dashboards/tempo-tenants.json @@ -1144,6 +1144,6 @@ }, "timezone": "utc", "title": "Tempo / Tenants", - "uid": "", + "uid": "tempo-coordinator-tenants", "version": 0 } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json index d510ffc..f0e3800 100644 --- a/src/grafana_dashboards/tempo-writes.json +++ b/src/grafana_dashboards/tempo-writes.json @@ -976,6 +976,6 @@ }, "timezone": "utc", "title": "Tempo / Writes", - "uid": "", + "uid": "tempo-coordinator-writes", "version": 0 } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index 9e2268a..dffae65 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -3713,7 +3713,7 @@ }, "timezone": "", "title": "Tempo Overview", - "uid": "a6175b9cc7ec20591890117c39580049", + "uid": "tempo-coordinator-overview", "version": 1, "weekStart": "" } \ No newline at end of file From e63b7c55a6129315c5618954ed85f6486c937885 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 21 Aug 2024 10:17:40 +0200 Subject: [PATCH 11/14] sum spans from different transport protocols --- src/grafana_dashboards/tempo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index dffae65..aff9d3e 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -354,7 +354,7 @@ "uid": "${prometheusds}" }, "editorMode": "builder", - "expr": "tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "expr": "sum(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", "interval": "", "legendFormat": "", "range": true, From 35057921162d766e9e883d7e9bc8ca21763edb5f Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Fri, 23 Aug 2024 14:57:30 +0200 Subject: [PATCH 12/14] Add explanations to dashboard rows --- src/grafana_dashboards/tempo-operational.json | 10819 ++++++++-------- src/grafana_dashboards/tempo-reads.json | 2692 ++-- src/grafana_dashboards/tempo-tenants.json | 2603 ++-- src/grafana_dashboards/tempo-writes.json | 2215 ++-- src/grafana_dashboards/tempo.json | 2877 ++-- 5 files changed, 11240 insertions(+), 9966 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index a2494bc..d3f74a9 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -1,5580 +1,5565 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [ - - ], - "type": "dashboard" - }, - "type": "dashboard" - }, - { - "datasource": { - "uid": "$logsds" - }, - "enable": true, - "expr": "{diff_{}container='kube-diff-logger'}", - "hide": true, - "iconColor": "rgba(255, 96, 96, 1)", - "name": "diffs", - "showIn": 0, - "target": { - - } - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo Dashboards", - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 38, - "panels": [ - - ], - "title": "General", - "type": "row" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "uid": "$logsds" + }, + "enable": true, + "expr": "{diff_{}container='kube-diff-logger'}", + "hide": true, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "diffs", + "showIn": 0, + "target": {} } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "rate(go_gc_duration_seconds_count{}[$__rate_interval])", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "gcs", - "type": "timeseries" }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 8, + "links": [ { - "expr": "go_memstats_heap_inuse_bytes{}", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "go heap", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ + ], + "liveNow": false, + "panels": [ { - "expr": "go_goroutines{}", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "goroutines", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 42, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "rate(container_cpu_usage_seconds_total{pod=~'$component.*', container!='POD'}[$__rate_interval])", - "interval": "", - "intervalFactor": 5, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "cpu", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 43, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "container_memory_working_set_bytes{pod=~'$component.*', container!='POD'}", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "working set", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 44, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 38, + "panels": [], + "title": "General", + "type": "row" }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ { - "expr": "rate(container_network_receive_bytes_total{pod=~'$component.*'}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "rx-{{pod}}", - "refId": "A" + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 114, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "General operational metrics. CPU metrics require a connected [node_exporter](https://github.com/prometheus/node_exporter), storage panel requires [kubelet metrics](https://kubernetes.io/docs/reference/instrumentation/metrics/) integration.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" }, { - "expr": "rate(container_network_transmit_bytes_total{pod=~'$component.*'}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "tx-{{pod}}", - "refId": "B" - } - ], - "title": "tx/rx", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 45, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "kubelet_volume_stats_available_bytes{persistentvolumeclaim=~'$component.*'}", - "legendFormat": "{{persistentvolumeclaim}}", - "refId": "A" - } - ], - "title": "data volume free", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 49, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 3 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 2 - }, - "id": 33, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "tempodb_work_queue_length{} / tempodb_work_queue_max{}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "%age total work queue", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(go_gc_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "gcs", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 2 - }, - "id": 32, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(increase(tempodb_compaction_errors_total{}[$__rate_interval])) by (job)", - "legendFormat": "compaction_err", - "refId": "B" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "sum(increase(tempodb_retention_errors_total{}[$__rate_interval])) by (job)", - "legendFormat": "retention_err", - "refId": "C" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 3 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{}[$__rate_interval])) by (job)", - "legendFormat": "blocklist_err", - "refId": "D" - } - ], - "title": "maintenance errors", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "go heap", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 2 - }, - "id": 35, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 3 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Blocklist Poll Duration", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "go_goroutines{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "legendFormat": "{{pod}}", + "refId": "A" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 2 - }, - "id": 47, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "avg(tempodb_blocklist_length{}) by (tenant)", - "instant": false, - "interval": "", - "legendFormat": "{{tenant}}", - "refId": "A" - } - ], - "title": "Blocklist Length", - "type": "timeseries" + ], + "title": "goroutines", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 2 - }, - "id": 51, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 3 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "retention duration", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{container!=\"POD\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "cpu", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 3 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 2 - }, - "id": 53, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(increase(tempodb_retention_deleted_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "deleted", - "refId": "A" }, - { - "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "marked_for_deletion", - "refId": "B" - } - ], - "title": "retention", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "container_memory_working_set_bytes{container!=\"POD\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "working set", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 3 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 18, - "y": 2 - }, - "id": 70, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "increase(kube_pod_container_status_restarts_total{}[5m]) > 0", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Container Restarts", - "type": "timeseries" - } - ], - "title": "Maintenance", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 21, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "rate(container_network_receive_bytes_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "rx-{{pod}}", + "refId": "A" + }, + { + "expr": "rate(container_network_transmit_bytes_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "tx-{{pod}}", + "refId": "B" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 13 - }, - "id": 90, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A" - } - ], - "title": "Queries/Sec (Query Frontend)", - "type": "timeseries" + ], + "title": "tx/rx", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 4, - "y": 13 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 3 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Query Latency (Query Frontend)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "kubelet_volume_stats_available_bytes{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",persistentvolumeclaim=~\"$component.*\"}", + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 13 - }, - "id": 98, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A", - "stepMode": "min" - } - ], - "title": "Search Queries/Sec (Query Frontend)", - "type": "timeseries" + ], + "title": "data volume free", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 13 - }, - "id": 94, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A", - "stepMode": "min" + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" }, - { - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B", - "stepMode": "min" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 }, - { - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C", - "stepMode": "min" - } - ], - "title": "Search Query Latency (Query Frontend)", - "type": "timeseries" + "id": 49, + "panels": [], + "title": "Maintenance", + "type": "row" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 18 - }, - "id": 91, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A" - } - ], - "title": "Queries/Sec (Querier)", - "type": "timeseries" + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 115, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Self-maintenance of the cluster. Look for gaps and sudden increases.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 4, - "y": 18 - }, - "id": 89, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 11 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Query Latency (Querier)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "tempodb_work_queue_length{} / tempodb_work_queue_max{}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "%age total work queue", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 11 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 18 - }, - "id": 99, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A", - "stepMode": "min" - } - ], - "title": "Search Queries/Sec (Querier)", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(increase(tempodb_compaction_errors_total{}[$__rate_interval])) by (job)", + "legendFormat": "compaction_err", + "refId": "B" + }, + { + "expr": "sum(increase(tempodb_retention_errors_total{}[$__rate_interval])) by (job)", + "legendFormat": "retention_err", + "refId": "C" + }, + { + "expr": "sum(increase(tempodb_blocklist_poll_errors_total{}[$__rate_interval])) by (job)", + "legendFormat": "blocklist_err", + "refId": "D" + } + ], + "title": "maintenance errors", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 18 - }, - "id": 95, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A", - "stepMode": "min" + "datasource": { + "uid": "${prometheusds}" }, - { - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B", - "stepMode": "min" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 11 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C", - "stepMode": "min" - } - ], - "title": "Search Query Latency (Querier)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Blocklist Poll Duration", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 11 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 23 - }, - "id": 92, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A" - } - ], - "title": "Queries/Sec (Ingester)", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "avg(tempodb_blocklist_length{}) by (tenant)", + "instant": false, + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Blocklist Length", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 4, - "y": 23 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 11 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Query Latency (Ingester)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "retention duration", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 11 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 23 - }, - "id": 100, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A", - "stepMode": "min" - } - ], - "title": "Search Queries/Sec (Ingester)", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(increase(tempodb_retention_deleted_total{}[$__rate_interval]))", + "interval": "", + "legendFormat": "deleted", + "refId": "A" + }, + { + "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{}[$__rate_interval]))", + "interval": "", + "legendFormat": "marked_for_deletion", + "refId": "B" + } + ], + "title": "retention", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 23 - }, - "id": 96, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A", - "stepMode": "min" + "datasource": { + "uid": "${prometheusds}" }, - { - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B", - "stepMode": "min" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 11 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C", - "stepMode": "min" - } - ], - "title": "Search Query Latency (Ingester)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{}[5m]) > 0", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Container Restarts", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 28 - }, - "id": 110, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { + "collapsed": false, + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics.*'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "range": true, - "refId": "A", - "stepMode": "min" - } - ], - "title": "Metrics Summary QPS (Query Frontend)", - "type": "timeseries" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 21, + "panels": [], + "title": "Read", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 28 - }, - "id": 111, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", + "datasource": { "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A", - "stepMode": "min" }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B", - "stepMode": "min" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C", - "stepMode": "min" - } - ], - "title": "Metrics Summary Latency (Query Frontend)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Queries/Sec (Query Frontend)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 33 - }, - "id": 112, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", + "datasource": { "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}} {{route}}", - "range": true, - "refId": "A" - } - ], - "title": "Metrics Summary QPS (Querier)", - "type": "timeseries" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 17 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Query Frontend)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 4, - "y": 33 - }, - "id": 113, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", + "datasource": { "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A" }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 17 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" - } - ], - "title": "Metrics Summary Latency (Querier)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (Query Frontend)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 33 - }, - "id": 106, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", + "datasource": { "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (status_code)", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "range": true, - "refId": "A" - } - ], - "title": "GetMetrics QPS (Metrics Generator)", - "type": "timeseries" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 17 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Query Frontend)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 16, - "y": 33 - }, - "id": 107, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", + "datasource": { "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A" }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 22 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" - } - ], - "title": "GetMetrics Latency (Metrics Generator)", - "type": "timeseries" - } - ], - "title": "Read", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 19, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 4 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Spans Received", - "type": "timeseries" + ], + "title": "Queries/Sec (Querier)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 4 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", + "datasource": { "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Traces Created", - "type": "timeseries" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 22 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Querier)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 22 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 4 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{}[1h]))", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Blocks Flushed (1h)", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (Querier)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 22 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 4 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Failed Flushes", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Querier)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 27 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 4 - }, - "id": 26, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Blocks Cleared (1h)", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Queries/Sec (Ingester)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 4 - }, - "id": 36, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 27 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Flush Duration", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Query Latency (Ingester)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 27 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 0, - "y": 9 - }, - "id": 72, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", - "interval": "", - "legendFormat": "accepted", - "refId": "A" }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", - "interval": "", - "legendFormat": "refused", - "refId": "B" - } - ], - "title": "OTEL Distributor Spans/second", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" + } + ], + "title": "Search Queries/Sec (Ingester)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 6, - "y": 14 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 27 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Push Latency (Ingester)", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" + } + ], + "title": "Search Query Latency (Ingester)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 19 - }, - "id": 109, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "accepted", - "range": true, - "refId": "A" }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", - "interval": "", - "legendFormat": "refused {{reason}}", - "range": true, - "refId": "B" - } - ], - "title": "Generator Spans/second", - "type": "timeseries" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 32 + }, + "id": 112, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}} {{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Metrics Summary QPS (Querier)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 6, - "y": 19 - }, - "id": 108, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A" }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 4, + "y": 32 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "datasource": { + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "Metrics Summary Latency (Querier)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" - } - ], - "title": "Push Latency (Generator)", - "type": "timeseries" - } - ], - "title": "Write", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 28, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 32 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{}[$__rate_interval])) by (status_code, operation)", - "interval": "", - "legendFormat": "{{status_code}}-{{operation}}", - "refId": "A" - } - ], - "title": "Requests/Second", - "type": "timeseries" + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics.*'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "range": true, + "refId": "A", + "stepMode": "min" + } + ], + "title": "Metrics Summary QPS (Query Frontend)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 30, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", - "legendFormat": ".99-{{operation}}", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", - "legendFormat": ".9-{{operation}}", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 32 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - { - "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", - "legendFormat": ".5-{{operation}}", - "refId": "C" - } - ], - "title": "Latency By Operation", - "type": "timeseries" - } - ], - "title": "Backend", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 56, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A", + "stepMode": "min" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B", + "stepMode": "min" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C", + "stepMode": "min" } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 17 - }, - "id": 62, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "tempo_memberlist_client_cluster_node_health_score{}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Node Health", - "type": "timeseries" + ], + "title": "Metrics Summary Latency (Query Frontend)", + "type": "timeseries" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 37 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 17 - }, - "id": 63, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "min(tempo_memberlist_client_cluster_members_count{})", - "interval": "", - "legendFormat": "min", - "refId": "A" }, - { - "expr": "max(tempo_memberlist_client_cluster_members_count{})", - "interval": "", - "legendFormat": "max", - "refId": "B" - } - ], - "title": "Member Count", - "type": "timeseries" + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "GetMetrics QPS (Metrics Generator)", + "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 17 - }, - "id": 60, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "expr": "min(tempo_memberlist_client_kv_store_count{})", - "interval": "", - "legendFormat": "min", - "refId": "A" }, - { - "datasource": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 16, + "y": 37 + }, + "id": 107, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "GetMetrics Latency (Metrics Generator)", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "expr": "max(tempo_memberlist_client_kv_store_count{})", - "interval": "", - "legendFormat": "max", - "refId": "B" - } - ], - "title": "KV Store Count", - "type": "timeseries" - } - ], - "title": "Ring/Memberlist", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 81, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 19, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 116, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Writes to Tempo backend. Look for gaps in receiving data or sudden spikes in latency.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 45 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Spans Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 45 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "builder", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Traces Created", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 45 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{}[1h]))", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Blocks Flushed (1h)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 45 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Failed Flushes", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 45 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Blocks Cleared (1h)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 45 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Flush Duration", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 50 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "refId": "A" + }, + { + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", + "interval": "", + "legendFormat": "refused", + "refId": "B" + } + ], + "title": "OTEL Distributor Spans/second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 6, + "y": 50 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Push Latency (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 6, + "y": 55 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "Push Latency (Generator)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 60 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", + "interval": "", + "legendFormat": "refused {{reason}}", + "range": true, + "refId": "B" + } + ], + "title": "Generator Spans/second", + "type": "timeseries" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 19 - }, - "id": 83, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_objects_combined_total{}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Objects Combined / s", - "type": "timeseries" + ], + "title": "Write", + "type": "row" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 117, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Backend HTTP response codes. Look for increase in 5xx/4xx status codes as well as latency spikes.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{}[$__rate_interval])) by (status_code, operation)", + "interval": "", + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "title": "Requests/Second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", + "legendFormat": ".5-{{operation}}", + "refId": "C" + } + ], + "title": "Latency By Operation", + "type": "timeseries" } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 19 - }, - "id": 85, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_objects_written_total{}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Objects Written / s", - "type": "timeseries" + ], + "title": "Backend", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 19 - }, - "id": 88, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { + "collapsed": true, + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum(rate(tempodb_compaction_bytes_written_total{}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Bytes Written / s", - "type": "timeseries" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 56, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 118, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "State of Tempo cluster. Look for unhealthy nodes and drops in number of members.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 47 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "tempo_memberlist_client_cluster_node_health_score{}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Node Health", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 47 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "min(tempo_memberlist_client_cluster_members_count{})", + "interval": "", + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "max(tempo_memberlist_client_cluster_members_count{})", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Member Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 47 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "expr": "min(tempo_memberlist_client_kv_store_count{})", + "interval": "", + "legendFormat": "min", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "expr": "max(tempo_memberlist_client_kv_store_count{})", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "KV Store Count", + "type": "timeseries" + } + ], + "title": "Ring/Memberlist", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 19 - }, - "id": 86, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { + "collapsed": true, + "datasource": { "type": "prometheus", "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(increase(tempodb_compaction_blocks_total{}[5m])) by (level)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Blocks Compacted", - "type": "timeseries" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 81, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 119, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "State of periodic compaction. Look for inconsistent gaps pattern.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 48 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_objects_combined_total{}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Objects Combined / s", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 48 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_objects_written_total{}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Objects Written / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 48 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "builder", + "expr": "sum(rate(tempodb_compaction_bytes_written_total{}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bytes Written / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 48 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(increase(tempodb_compaction_blocks_total{}[5m])) by (level)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Blocks Compacted", + "type": "timeseries" + } + ], + "title": "Compactor", + "type": "row" } - ], - "title": "Compactor", - "type": "row" - } - ], - "refresh": "", - "schemaVersion": 38, - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "ds", - "options": [ - - ], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "loki-ops", - "value": "loki-ops" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "logsds", - "options": [ - - ], - "query": "loki", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": "ops-us-east-0", - "value": "ops-us-east-0" - }, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "definition": "label_values(tempo_build_info, cluster)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": { - "query": "label_values(tempo_build_info, cluster)", - "refId": "ops-cortex-cluster-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": "tempo-ops", - "value": "tempo-ops" - }, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "definition": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": { - "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", - "refId": "ops-cortex-namespace-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "hide": 0, - "includeAll": true, - "multi": false, - "name": "component", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "compactor", - "value": "compactor" - }, - { - "selected": false, - "text": "distributor", - "value": "distributor" - }, - { - "selected": false, - "text": "ingester", - "value": "ingester" - }, - { - "selected": false, - "text": "metrics-generator", - "value": "metrics-generator" - }, - { - "selected": false, - "text": "query-frontend", - "value": "query-frontend" - }, - { - "selected": false, - "text": "querier", - "value": "querier" - } - ], - "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Tempo Operational", - "uid": "tempo-coordinator-operational", - "version": 1, - "weekStart": "" + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Loki datasource", + "multi": true, + "name": "lokids", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Prometheus datasource", + "multi": true, + "name": "prometheusds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "hide": 0, + "includeAll": true, + "label": "Juju unit", + "multi": true, + "name": "juju_unit", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "hide": 0, + "includeAll": true, + "label": "Juju application", + "multi": true, + "name": "juju_application", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "hide": 0, + "includeAll": true, + "label": "Juju model uuid", + "multi": true, + "name": "juju_model_uuid", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up,juju_model)", + "hide": 0, + "includeAll": true, + "label": "Juju model", + "multi": true, + "name": "juju_model", + "options": [], + "query": { + "query": "label_values(up,juju_model)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_loki_0", + "value": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_loki_0" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "logsds", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(tempo_build_info, cluster)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(tempo_build_info, cluster)", + "refId": "ops-cortex-cluster-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refId": "ops-cortex-namespace-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "multi": false, + "name": "component", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "compactor", + "value": "compactor" + }, + { + "selected": false, + "text": "distributor", + "value": "distributor" + }, + { + "selected": false, + "text": "ingester", + "value": "ingester" + }, + { + "selected": false, + "text": "metrics-generator", + "value": "metrics-generator" + }, + { + "selected": false, + "text": "query-frontend", + "value": "query-frontend" + }, + { + "selected": false, + "text": "querier", + "value": "querier" + } + ], + "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Tempo Operational", + "uid": "tempo-coordinator-operational", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json index ae5aff4..9ad724a 100644 --- a/src/grafana_dashboards/tempo-reads.json +++ b/src/grafana_dashboards/tempo-reads.json @@ -1,1192 +1,1612 @@ { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo Dashboards", - "type": "dashboards" - } - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ + "annotations": { + "list": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 15, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Query Frontend", + "type": "row" }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval])) by (route)", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "id": 20, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Query Frontend is responsible for sharding incoming requests for faster processing in parallel (by the queriers). See [Query Frontend](https://grafana.com/docs/tempo/latest/configuration/#query-frontend) for more details on the component. The panels show queries per second and their latency.", + "mode": "markdown" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Query Frontend", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'querier_api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval])) by (route)", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Querier", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 16, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Querier", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 21, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The Querier is responsible for querying the backends/cache for the traceID. See [Querier](https://grafana.com/docs/tempo/latest/configuration/#querier) for more details on the component. The panels show queries per second and their latency.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'querier_api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 }, - { - "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval])) by (endpoint)", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 + "id": 17, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Querier External Endpoint", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "External endpoints are used to offload backend search requests. If they aren't enabled, the panels won't show any data. See [Serverless](https://grafana.com/docs/tempo/latest/operations/backend_search/#serverless-environment) for more information on external endpoints.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval])) by (endpoint)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Querier External Endpoint", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 18, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Ingester", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 23, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The ingester is responsible for batching up traces and pushing them to TempoDB. See [Ingester](https://grafana.com/docs/tempo/latest/configuration/#ingester) for more information.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route)", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 + "id": 19, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Backend", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 24, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Backend metrics show the overall state of Tempo's backend components.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 43 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 43 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation='GET'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Loki datasource", + "multi": true, + "name": "lokids", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Prometheus datasource", + "multi": true, + "name": "prometheusds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "hide": 0, + "includeAll": true, + "label": "Juju unit", + "multi": true, + "name": "juju_unit", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "hide": 0, + "includeAll": true, + "label": "Juju application", + "multi": true, + "name": "juju_application", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "hide": 0, + "includeAll": true, + "label": "Juju model uuid", + "multi": true, + "name": "juju_model_uuid", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation='GET'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up,juju_model)", + "hide": 0, + "includeAll": true, + "label": "Juju model", + "multi": true, + "name": "juju_model", + "options": [], + "query": { + "query": "label_values(up,juju_model)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Backend", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "cluster", - "multi": true, - "name": "cluster", - "options": [ - - ], - "query": "label_values(tempo_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": true, - "name": "namespace", - "options": [ - - ], - "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Reads", - "uid": "tempo-coordinator-reads", - "version": 0 + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Reads", + "uid": "tempo-coordinator-reads", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-tenants.json b/src/grafana_dashboards/tempo-tenants.json index 7c544e6..62e2b74 100644 --- a/src/grafana_dashboards/tempo-tenants.json +++ b/src/grafana_dashboards/tempo-tenants.json @@ -1,1149 +1,1564 @@ { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo Dashboards", - "type": "dashboards" - } - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, + "annotations": { + "list": [ { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - ], - "targets": [ - { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{})\n) by (limit_name)\n", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Limits", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Tenant info", - "titleSize": "h6" + ] }, - { - "collapse": false, - "height": "250px", - "panels": [ + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 10, + "links": [ { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "dashes": true, - "fill": 0 + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "alias": "burst limit", - "dashes": true, - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_distributor_bytes_received_total{tenant='$tenant'}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "received", - "legendLink": null + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 }, - { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_rate_limit_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_rate_limit_bytes'})\n) by (ingestion_rate_limit_bytes)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null + "id": 11, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Tenant info", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_burst_size_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_burst_size_bytes'})\n) by (ingestion_burst_size_bytes)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "burst limit", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Distributor bytes/s", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "id": 16, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Tenant configuration and limits for the specified tenant. ", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_distributor_spans_received_total{tenant='$tenant'}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "accepted", - "legendLink": null + "aliasColors": {}, + "bars": false, + "columns": [], + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "expr": "sum(rate(tempo_discarded_spans_total{tenant='$tenant'}[$__rate_interval])) by (reason)", - "format": "time_series", - "interval": "1m", - "legendFormat": "refused {{ reason }}", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Distributor spans/s", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{})\n) by (limit_name)\n", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "title": "Limits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table-old", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Ingestion", + "type": "row" }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "global limit", - "dashes": true, - "fill": 0 + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "alias": "local limit", - "dashes": true, - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(tempo_ingester_live_traces{tenant='$tenant'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "live traces", - "legendLink": null + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 11 }, - { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_global_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_global_traces_per_user'})\n) by (max_global_traces_per_user)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "global limit", - "legendLink": null + "id": 17, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Ingestion statistics for the specified tenant. Look for spikes or gaps in ingestion.", + "mode": "markdown" }, - { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_local_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_local_traces_per_user'})\n) by (max_local_traces_per_user)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "local limit", - "legendLink": null + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "dashes": true, + "fill": 0 + }, + { + "alias": "burst limit", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_distributor_bytes_received_total{tenant='$tenant'}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "received", + "refId": "A" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_rate_limit_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_rate_limit_bytes'})\n) by (ingestion_rate_limit_bytes)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "refId": "B" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_burst_size_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_burst_size_bytes'})\n) by (ingestion_burst_size_bytes)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "burst limit", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Distributor bytes/s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Live traces", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 13 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total{tenant='$tenant'}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "accepted", + "refId": "A" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_discarded_spans_total{tenant='$tenant'}[$__rate_interval])) by (reason)", + "format": "time_series", + "interval": "1m", + "legendFormat": "refused {{ reason }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Distributor spans/s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingestion", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='traces'}[$__rate_interval])) by (status)", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{ status }}", - "legendLink": null + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 13 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "global limit", + "dashes": true, + "fill": 0 + }, + { + "alias": "local limit", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(tempo_ingester_live_traces{tenant='$tenant'})", + "format": "time_series", + "interval": "1m", + "legendFormat": "live traces", + "refId": "A" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_global_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_global_traces_per_user'})\n) by (max_global_traces_per_user)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "global limit", + "refId": "B" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_local_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_local_traces_per_user'})\n) by (max_local_traces_per_user)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "local limit", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Live traces", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Queries/s (ID lookup)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 13, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Reads", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 18, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Traces data reads for the specified tenant. Look for spikes or gaps in ingestion.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='traces'}[$__rate_interval])) by (status)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Queries/s (ID lookup)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='search'}[$__rate_interval])) by (status)", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{ status }}", - "legendLink": null + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='search'}[$__rate_interval])) by (status)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Queries/s (search)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Queries/s (search)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 14, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Storage", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 19, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Storage data behaviour for the specified tenant.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "avg(tempodb_blocklist_length{tenant='$tenant'})", + "format": "time_series", + "interval": "1m", + "legendFormat": "length", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Blockslist length", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Reads", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 7, - "legend": { - "show": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tempodb_blocklist_length{tenant='$tenant'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "length", - "legendLink": null + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(tempodb_compaction_outstanding_blocks{tenant='$tenant'})\n/\ncount(tempo_build_info{})\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "blocks", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Outstanding compactions", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Blockslist length", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Metrics generator", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 20, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Tenant metric generator data. Note that for any data to show up, metrics-generator needs to be show up. See [its documentation](https://grafana.com/docs/tempo/latest/metrics-generator/) for more details on the component.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 43 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{tenant='$tenant'}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bytes/s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 8, - "legend": { - "show": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tempodb_compaction_outstanding_blocks{tenant='$tenant'})\n/\ncount(tempo_build_info{})\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "blocks", - "legendLink": null + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 43 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(tempo_metrics_generator_registry_active_series{tenant='$tenant'})", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{ tenant }}", + "refId": "A" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='metrics_generator_max_active_series'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='metrics_generator_max_active_series'})\n) by (metrics_generator_max_active_series)\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "limit", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Outstanding compactions", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Loki datasource", + "multi": true, + "name": "lokids", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Storage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 9, - "legend": { - "show": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Prometheus datasource", + "multi": true, + "name": "prometheusds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { - "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{tenant='$tenant'}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "rate", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Bytes/s", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "hide": 0, + "includeAll": true, + "label": "Juju unit", + "multi": true, + "name": "juju_unit", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "hide": 0, + "includeAll": true, + "label": "Juju application", + "multi": true, + "name": "juju_application", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "hide": 0, + "includeAll": true, + "label": "Juju model uuid", + "multi": true, + "name": "juju_model_uuid", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "alias": "limit", - "dashes": true, - "fill": 0 - } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up,juju_model)", + "hide": 0, + "includeAll": true, + "label": "Juju model", + "multi": true, + "name": "juju_model", + "options": [], + "query": { + "query": "label_values(up,juju_model)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "expr": "sum(tempo_metrics_generator_registry_active_series{tenant='$tenant'})", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{ tenant }}", - "legendLink": null + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" }, { - "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='metrics_generator_max_active_series'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='metrics_generator_max_active_series'})\n) by (metrics_generator_max_active_series)\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "limit", - "legendLink": null - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Active series", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(tempo_build_info{}, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "current": { + "selected": false, + "text": "single-tenant", + "value": "single-tenant" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "tenant", + "multi": false, + "name": "tenant", + "options": [], + "query": "label_values(tempodb_blocklist_length{}, tenant)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Metrics generator", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "cluster", - "multi": true, - "name": "cluster", - "options": [ - - ], - "query": "label_values(tempo_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": true, - "name": "namespace", - "options": [ - - ], - "query": "label_values(tempo_build_info{}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": false, - "label": "tenant", - "multi": false, - "name": "tenant", - "options": [ - - ], - "query": "label_values(tempodb_blocklist_length{}, tenant)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Tenants", - "uid": "tempo-coordinator-tenants", - "version": 0 + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Tenants", + "uid": "tempo-coordinator-tenants", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json index f0e3800..0dbd705 100644 --- a/src/grafana_dashboards/tempo-writes.json +++ b/src/grafana_dashboards/tempo-writes.json @@ -1,981 +1,1328 @@ { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "tempo" - ], - "targetBlank": false, - "title": "Tempo Dashboards", - "type": "dashboards" - } - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ + "annotations": { + "list": [ { - "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "accepted", - "legendLink": null - }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", - "format": "time_series", - "interval": "1m", - "legendFormat": "refused", - "legendLink": null + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Spans/Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Distributor", + "type": "row" }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 }, - { - "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Distributors receive spans and forward them to the appropriate ingesters. See [Distributor](https://grafana.com/docs/tempo/latest/configuration/#distributor) for more details on the component.", + "mode": "markdown" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Distributor", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "accepted", + "refId": "A" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", + "format": "time_series", + "interval": "1m", + "legendFormat": "refused", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Spans/Second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 }, - { - "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 + "id": 18, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Ingester", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 21, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The ingester is responsible for batching up traces and pushing them to TempoDB. See [Ingester](https://grafana.com/docs/tempo/latest/configuration/#ingester) for more details on the component.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 19, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Backend - Ingester", + "type": "row" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 }, - { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 + "id": 20, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Backend - Compactor", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 23, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Compactors stream blocks from the storage backend, combine them and write them back. See [Compactor](https://grafana.com/docs/tempo/latest/configuration/#compactor) for more details on the component.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${prometheusds}" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 99th", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} 50th", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{route}} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ] } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Backend - Ingester", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 10, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "tempo" + ], + "templating": { + "list": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n", - "format": "time_series", - "interval": "1m", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Loki datasource", + "multi": true, + "name": "lokids", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Prometheus datasource", + "multi": true, + "name": "prometheusds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${prometheusds}", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "hide": 0, + "includeAll": true, + "label": "Juju unit", + "multi": true, + "name": "juju_unit", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, { - "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 99th", - "refId": "A", - "step": 10 + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "hide": 0, + "includeAll": true, + "label": "Juju application", + "multi": true, + "name": "juju_application", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} 50th", - "refId": "B", - "step": 10 + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "hide": 0, + "includeAll": true, + "label": "Juju model uuid", + "multi": true, + "name": "juju_model_uuid", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()", - "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{route}} Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up,juju_model)", + "hide": 0, + "includeAll": true, + "label": "Juju model", + "multi": true, + "name": "juju_model", + "options": [], + "query": { + "query": "label_values(up,juju_model)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(tempo_build_info, cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(tempo_build_info{}, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Backend - Compactor", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "tempo" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "cluster", - "multi": true, - "name": "cluster", - "options": [ - - ], - "query": "label_values(tempo_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "${prometheusds}", - "hide": 0, - "includeAll": true, - "label": "namespace", - "multi": true, - "name": "namespace", - "options": [ - - ], - "query": "label_values(tempo_build_info{}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Tempo / Writes", - "uid": "tempo-coordinator-writes", - "version": 0 + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Tempo / Writes", + "uid": "tempo-coordinator-writes", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index aff9d3e..76b463c 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -65,12 +65,35 @@ "y": 0 }, "id": 312, - "panels": [ - - ], + "panels": [], "title": "At a glance", "type": "row" }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 324, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "High-level overview of core tracing system metrics: amount of traces, spans.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", @@ -81,37 +104,6 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -134,28 +126,34 @@ "h": 5, "w": 3, "x": 0, - "y": 0 + "y": 3 }, "id": 320, "options": { - "reduceOptions": { - "values": false, - "calcs": [ - "last" - ], - "fields": "/^version$/" - }, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "/^version$/", + "values": false + }, + "textMode": "auto", "tooltip": { "mode": "multi", "sort": "none" } }, + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -184,37 +182,6 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -237,21 +204,34 @@ "h": 5, "w": 7, "x": 3, - "y": 0 + "y": 3 }, "id": 321, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", "tooltip": { "mode": "multi", "sort": "none" } }, + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -279,37 +259,6 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -332,21 +281,34 @@ "h": 5, "w": 7, "x": 10, - "y": 0 + "y": 3 }, "id": 322, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", "tooltip": { "mode": "multi", "sort": "none" } }, + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -373,37 +335,6 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -426,21 +357,34 @@ "h": 5, "w": 7, "x": 17, - "y": 0 + "y": 3 }, "id": 323, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", "tooltip": { "mode": "multi", "sort": "none" } }, + "pluginVersion": "9.5.3", "targets": [ { "expr": "tempo_ingester_flush_size_bytes_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", @@ -457,12 +401,37 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 8 }, "id": 134, "title": "General", "type": "row" }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 325, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Quick overview of main metrics: CPU usage, traces ingestion, flush duration. Use for verifying if spans are ingested by the system.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", @@ -524,7 +493,7 @@ "h": 8, "w": 16, "x": 0, - "y": 7 + "y": 11 }, "id": 132, "options": { @@ -617,7 +586,7 @@ "h": 8, "w": 8, "x": 16, - "y": 7 + "y": 11 }, "id": 130, "options": { @@ -640,7 +609,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_ingester_flush_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -652,7 +621,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_ingester_flush_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -664,7 +633,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_ingester_flush_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -738,7 +707,7 @@ "h": 8, "w": 8, "x": 0, - "y": 15 + "y": 19 }, "id": 136, "options": { @@ -842,7 +811,7 @@ "h": 8, "w": 8, "x": 8, - "y": 15 + "y": 19 }, "id": 117, "options": { @@ -864,7 +833,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by (juju_unit) (rate(tempo_ingester_flush_duration_seconds_sum{juju_application=~\"$juju_application\", juju_model=~\"$juju_model\", juju_model_uuid=~\"$juju_model_uuid\", juju_unit=~\"$juju_unit\"}[5m]))\n", + "expr": "sum by(juju_unit) (rate(tempo_ingester_flush_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "legendFormat": "__auto", "range": true, "refId": "A" @@ -935,7 +904,7 @@ "h": 8, "w": 8, "x": 16, - "y": 15 + "y": 19 }, "id": 139, "options": { @@ -958,7 +927,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_distributor_push_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -970,7 +939,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_distributor_push_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -981,7 +950,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_distributor_push_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -999,12 +968,37 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 27 }, "id": 129, "title": "Network", "type": "row" }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 326, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Overview of main metrics: CPU usage, traces ingestion, flush duration. Use for verifying if spans are ingested by the system.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", @@ -1067,7 +1061,7 @@ "h": 8, "w": 18, "x": 0, - "y": 24 + "y": 30 }, "id": 78, "options": { @@ -1090,7 +1084,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1102,7 +1096,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1114,7 +1108,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1188,7 +1182,7 @@ "h": 8, "w": 6, "x": 18, - "y": 24 + "y": 30 }, "id": 119, "options": { @@ -1226,12 +1220,37 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 38 }, "id": 123, "title": "Backend", "type": "row" }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 327, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Backend: storage usage, compaction. Look for any abnormalities in compacting data and retention.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", @@ -1293,7 +1312,7 @@ "h": 8, "w": 18, "x": 0, - "y": 33 + "y": 41 }, "id": 121, "options": { @@ -1388,7 +1407,7 @@ "h": 8, "w": 6, "x": 18, - "y": 33 + "y": 41 }, "id": 51, "options": { @@ -1411,7 +1430,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1423,7 +1442,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1435,7 +1454,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1510,7 +1529,7 @@ "h": 8, "w": 6, "x": 0, - "y": 41 + "y": 49 }, "id": 125, "options": { @@ -1605,7 +1624,7 @@ "h": 8, "w": 6, "x": 6, - "y": 41 + "y": 49 }, "id": 127, "options": { @@ -1700,7 +1719,7 @@ "h": 8, "w": 6, "x": 12, - "y": 41 + "y": 49 }, "id": 53, "options": { @@ -1722,7 +1741,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum(increase(tempodb_retention_deleted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", @@ -1734,7 +1753,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", @@ -1809,7 +1828,7 @@ "h": 8, "w": 6, "x": 18, - "y": 41 + "y": 49 }, "id": 35, "options": { @@ -1832,7 +1851,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1844,7 +1863,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1855,7 +1874,7 @@ { "datasource": { "type": "prometheus", - "uid": "P0899610D3ECDF9EC" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", @@ -1878,13 +1897,38 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 57 }, "id": 38, "panels": [], - "title": "CPU / Memory / Goroutines", + "title": "Memory / Goroutines", "type": "row" }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 328, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Memory usage and garbage collection from Go perspective.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "uid": "${prometheusds}" @@ -1908,7 +1952,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1947,7 +1990,7 @@ "h": 5, "w": 6, "x": 0, - "y": 50 + "y": 60 }, "id": 24, "options": { @@ -1997,7 +2040,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2036,7 +2078,7 @@ "h": 5, "w": 6, "x": 6, - "y": 50 + "y": 60 }, "id": 25, "options": { @@ -2086,7 +2128,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2125,7 +2166,7 @@ "h": 5, "w": 6, "x": 12, - "y": 50 + "y": 60 }, "id": 23, "options": { @@ -2174,7 +2215,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2213,7 +2253,7 @@ "h": 5, "w": 6, "x": 18, - "y": 50 + "y": 60 }, "id": 43, "options": { @@ -2250,1264 +2290,1331 @@ "h": 1, "w": 24, "x": 0, - "y": 55 + "y": 65 }, "id": 49, - "panels": [ - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 2 - }, - "id": 33, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "tempodb_work_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} / tempodb_work_queue_max{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "% of work queue filled", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 2 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", - "legendFormat": "compaction_err", - "refId": "B" - }, - { - "expr": "sum(increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", - "legendFormat": "retention_err", - "refId": "C" - }, - { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", - "legendFormat": "blocklist_err", - "refId": "D" - } - ], - "title": "Maintenance Errors", - "type": "timeseries" + "panels": [], + "title": "Maintenance", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 329, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Potential maintenance issues. Look for increases in queues length, errors, restarts as they may mean resources are underprovisioned.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 68 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "tempodb_work_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} / tempodb_work_queue_max{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "% of work queue filled", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 68 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "sum(increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", + "legendFormat": "compaction_err", + "refId": "B" + }, + { + "expr": "sum(increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", + "legendFormat": "retention_err", + "refId": "C" + }, + { + "expr": "sum(increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", + "legendFormat": "blocklist_err", + "refId": "D" + } + ], + "title": "Maintenance Errors", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 68 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "exemplar": true, + "expr": "avg(tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) by (tenant)", + "instant": false, + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "title": "Blocklist Length", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 68 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]) > 0", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Container Restarts", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 19, + "panels": [], + "title": "Write", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 330, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Writing new spans into storage. Look for gaps or sudden increases in latency.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 76 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Spans Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 76 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "builder", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Traces Created", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 76 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Blocks Flushed (1h)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 76 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Blocks Cleared (1h)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ { - "datasource": { - "uid": "${prometheusds}" + "expr": "sum(rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", + "interval": "", + "legendFormat": "accepted", + "refId": "A" + }, + { + "expr": "sum(rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", + "interval": "", + "legendFormat": "refused", + "refId": "B" + } + ], + "title": "OTEL Distributor Spans/second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 2 - }, - "id": 47, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "exemplar": true, - "expr": "avg(tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) by (tenant)", - "instant": false, - "interval": "", - "legendFormat": "{{tenant}}", - "refId": "A" - } - ], - "title": "Blocklist Length", - "type": "timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" }, { - "datasource": { - "uid": "${prometheusds}" + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Push Latency (Gateway)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 2 - }, - "id": 70, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "increase(kube_pod_container_status_restarts_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]) > 0", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Container Restarts", - "type": "timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 86 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" } ], - "title": "Maintenance", + "title": "Push Latency (Ingester)", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 81, + "panels": [], + "title": "Compactor", "type": "row" }, { - "collapsed": true, "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "type": "datasource", + "uid": "grafana" }, "gridPos": { - "h": 1, + "h": 2, "w": 24, "x": 0, - "y": 56 + "y": 92 }, - "id": 19, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Spans Received", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 4 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Traces Created", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 4 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Blocks Flushed (1h)", - "type": "timeseries" + "id": 331, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 4 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "content": "Compaction is expected to happen periodically. Blocks should be compacted to higher levels. Look for any irregularities that don't match the patterns observed before.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))", - "interval": "", - "legendFormat": "", - "refId": "A" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Blocks Cleared (1h)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 72, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", - "interval": "", - "legendFormat": "accepted", - "refId": "A" - }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", - "interval": "", - "legendFormat": "refused", - "refId": "B" - } - ], - "title": "OTEL Distributor Spans/second", - "type": "timeseries" + "unit": "short" }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 79, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Push Latency (Gateway)", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 94 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "title": "Push Latency (Ingester)", - "type": "timeseries" + "editorMode": "builder", + "expr": "sum(rate(tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], - "title": "Write", - "type": "row" + "title": "Bytes Written / s", + "type": "timeseries" }, { - "collapsed": true, "datasource": { "type": "prometheus", "uid": "${prometheusds}" }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 81, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 61 - }, - "id": 88, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum(rate(tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Bytes Written / s", - "type": "timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 94 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 61 - }, - "id": 86, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(increase(tempodb_compaction_blocks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Blocks Compacted", - "type": "timeseries" + "editorMode": "code", + "expr": "sum(increase(tempodb_compaction_blocks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], - "title": "Compactor", - "type": "row" + "title": "Blocks Compacted", + "type": "timeseries" } ], - "refresh": false, - "schemaVersion": 37, + "refresh": "", + "schemaVersion": 38, "style": "dark", "tags": [ "tempo" From 9abdde8bb70f2531fe44b136ed431bd3ef801fe8 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Mon, 26 Aug 2024 12:34:32 +0200 Subject: [PATCH 13/14] Reordering and removal of several panels on tempo-operational and tempo overview --- src/grafana_dashboards/tempo-operational.json | 1766 ++++++----------- src/grafana_dashboards/tempo.json | 139 +- 2 files changed, 584 insertions(+), 1321 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index d3f74a9..1ecbcf1 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -36,7 +36,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 8, + "id": 36, "links": [ { "asDropdown": true, @@ -835,7 +835,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "tempodb_work_queue_length{} / tempodb_work_queue_max{}", + "expr": "tempodb_work_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} / tempodb_work_queue_max{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", "legendFormat": "{{instance}}", "refId": "A" } @@ -922,17 +922,17 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_compaction_errors_total{}[$__rate_interval])) by (job)", + "expr": "sum by(job) (increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "compaction_err", "refId": "B" }, { - "expr": "sum(increase(tempodb_retention_errors_total{}[$__rate_interval])) by (job)", + "expr": "sum by(job) (increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "retention_err", "refId": "C" }, { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{}[$__rate_interval])) by (job)", + "expr": "sum by(job) (increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "blocklist_err", "refId": "D" } @@ -1019,18 +1019,18 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1119,7 +1119,7 @@ "targets": [ { "exemplar": true, - "expr": "avg(tempodb_blocklist_length{}) by (tenant)", + "expr": "avg by(tenant) (tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", "instant": false, "interval": "", "legendFormat": "{{tenant}}", @@ -1208,19 +1208,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1308,13 +1308,13 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_retention_deleted_total{}[$__rate_interval]))", + "expr": "sum(increase(tempodb_retention_deleted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "interval": "", "legendFormat": "deleted", "refId": "A" }, { - "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{}[$__rate_interval]))", + "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "interval": "", "legendFormat": "marked_for_deletion", "refId": "B" @@ -1402,7 +1402,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{}[5m]) > 0", + "expr": "increase(kube_pod_container_status_restarts_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]) > 0", "hide": false, "interval": "", "legendFormat": "{{pod}}", @@ -1429,6 +1429,31 @@ "title": "Read", "type": "row" }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 120, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Reads by component of Tempo cluster. Look for gaps and sudden increases.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "uid": "${prometheusds}" @@ -1490,7 +1515,7 @@ "h": 5, "w": 4, "x": 0, - "y": 17 + "y": 19 }, "id": 90, "options": { @@ -1508,7 +1533,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_traces_traceid'}[$__rate_interval])) by (status_code)", + "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1579,7 +1604,7 @@ "h": 5, "w": 6, "x": 4, - "y": 17 + "y": 19 }, "id": 17, "options": { @@ -1597,19 +1622,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1678,8 +1703,8 @@ "gridPos": { "h": 5, "w": 4, - "x": 12, - "y": 17 + "x": 10, + "y": 19 }, "id": 98, "options": { @@ -1698,7 +1723,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_search.*'}[$__rate_interval])) by (status_code)", + "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1769,8 +1794,8 @@ "gridPos": { "h": 5, "w": 6, - "x": 16, - "y": 17 + "x": 14, + "y": 19 }, "id": 94, "options": { @@ -1789,7 +1814,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -1797,7 +1822,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -1805,7 +1830,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -1876,7 +1901,7 @@ "h": 5, "w": 4, "x": 0, - "y": 22 + "y": 24 }, "id": 91, "options": { @@ -1894,7 +1919,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (status_code)", + "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1965,7 +1990,7 @@ "h": 5, "w": 6, "x": 4, - "y": 22 + "y": 24 }, "id": 89, "options": { @@ -1983,19 +2008,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_traces_traceid'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2064,8 +2089,8 @@ "gridPos": { "h": 5, "w": 4, - "x": 12, - "y": 22 + "x": 10, + "y": 24 }, "id": 99, "options": { @@ -2084,7 +2109,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_search.*'}[$__rate_interval])) by (status_code)", + "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2155,8 +2180,8 @@ "gridPos": { "h": 5, "w": 6, - "x": 16, - "y": 22 + "x": 14, + "y": 24 }, "id": 95, "options": { @@ -2175,7 +2200,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2183,7 +2208,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2191,7 +2216,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -2262,7 +2287,7 @@ "h": 5, "w": 4, "x": 0, - "y": 27 + "y": 29 }, "id": 92, "options": { @@ -2280,7 +2305,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (status_code)", + "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2351,7 +2376,7 @@ "h": 5, "w": 6, "x": 4, - "y": 27 + "y": 29 }, "id": 3, "options": { @@ -2369,19 +2394,19 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.Querier/FindTraceByID'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2450,8 +2475,8 @@ "gridPos": { "h": 5, "w": 4, - "x": 12, - "y": 27 + "x": 10, + "y": 29 }, "id": 100, "options": { @@ -2470,7 +2495,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (status_code)", + "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval]))", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -2541,8 +2566,8 @@ "gridPos": { "h": 5, "w": 6, - "x": 16, - "y": 27 + "x": 14, + "y": 29 }, "id": 96, "options": { @@ -2561,7 +2586,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".99", "refId": "A", @@ -2569,7 +2594,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".9", "refId": "B", @@ -2577,7 +2602,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/Search.*'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])))", "interval": "", "legendFormat": ".5", "refId": "C", @@ -2587,6 +2612,48 @@ "title": "Search Query Latency (Ingester)", "type": "timeseries" }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 19, + "panels": [], + "title": "Write", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 116, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Writes to Tempo backend. Look for gaps in receiving data or sudden spikes in latency.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", @@ -2604,7 +2671,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2618,10 +2685,10 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2647,24 +2714,24 @@ }, "gridPos": { "h": 5, - "w": 4, + "w": 3, "x": 0, - "y": 32 + "y": 37 }, - "id": 112, + "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { "datasource": { @@ -2672,15 +2739,14 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (status_code)", - "hide": false, + "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", "interval": "", - "legendFormat": "{{status_code}} {{route}}", + "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Metrics Summary QPS (Querier)", + "title": "Spans Received", "type": "timeseries" }, { @@ -2714,10 +2780,10 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2737,74 +2803,49 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, - "w": 6, - "x": 4, - "y": 32 + "w": 3, + "x": 3, + "y": 37 }, - "id": 113, + "id": 9, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheusds}" }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", + "editorMode": "builder", + "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", "interval": "", - "legendFormat": ".99", + "legendFormat": "", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" } ], - "title": "Metrics Summary Latency (Querier)", + "title": "Traces Created", "type": "timeseries" }, { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -2833,7 +2874,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -2862,47 +2903,37 @@ }, "gridPos": { "h": 5, - "w": 4, - "x": 12, - "y": 32 + "w": 3, + "x": 6, + "y": 37 }, - "id": 110, + "id": 8, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~'.*api_metrics.*'}[$__rate_interval])) by (status_code)", - "hide": false, + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{}[1h]))", "interval": "", - "legendFormat": "{{status_code}}", - "range": true, - "refId": "A", - "stepMode": "min" + "legendFormat": "{{pod}}", + "refId": "A" } ], - "title": "Metrics Summary QPS (Query Frontend)", + "title": "Blocks Flushed (1h)", "type": "timeseries" }, { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -2931,7 +2962,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -2954,80 +2985,43 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, - "w": 6, - "x": 16, - "y": 32 + "w": 3, + "x": 9, + "y": 37 }, - "id": 111, + "id": 12, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A", - "stepMode": "min" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B", - "stepMode": "min" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'.*api_metrics.*'}[$__rate_interval])) by (le))", + "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C", - "stepMode": "min" + "legendFormat": "{{pod}}", + "refId": "A" } ], - "title": "Metrics Summary Latency (Query Frontend)", + "title": "Failed Flushes", "type": "timeseries" }, { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -3056,7 +3050,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -3085,45 +3079,37 @@ }, "gridPos": { "h": 5, - "w": 4, + "w": 3, "x": 12, "y": 37 }, - "id": 106, + "id": 26, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_request_duration_seconds_count{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (status_code)", - "hide": false, + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", "interval": "", - "legendFormat": "{{status_code}}", - "range": true, + "legendFormat": "", "refId": "A" } ], - "title": "GetMetrics QPS (Metrics Generator)", + "title": "Blocks Cleared (1h)", "type": "timeseries" }, { "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, "fieldConfig": { @@ -3137,7 +3123,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { @@ -3147,12 +3133,12 @@ }, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "always", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -3181,11 +3167,11 @@ }, "gridPos": { "h": 5, - "w": 6, - "x": 16, + "w": 3, + "x": 15, "y": 37 }, - "id": 107, + "id": 36, "options": { "legend": { "calcs": [], @@ -3198,1064 +3184,449 @@ "sort": "none" } }, - "pluginVersion": "9.0.0-d373beebpre", + "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", - "range": true, "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", - "range": true, "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route='/tempopb.MetricsGenerator/GetMetrics'}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", - "range": true, "refId": "C" } ], - "title": "GetMetrics Latency (Metrics Generator)", + "title": "Flush Duration", "type": "timeseries" }, { - "collapsed": true, "datasource": { - "type": "prometheus", "uid": "${prometheusds}" }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 19, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 116, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "content": "Writes to Tempo backend. Look for gaps in receiving data or sudden spikes in latency.", - "mode": "markdown" - }, - "pluginVersion": "9.5.3", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 45 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Spans Received", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "refId": "A" + }, + { + "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", + "interval": "", + "legendFormat": "refused", + "refId": "B" + } + ], + "title": "OTEL Distributor Spans/second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 45 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Traces Created", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 45 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + { + "color": "red", + "value": 80 + } + ] }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{}[1h]))", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Blocks Flushed (1h)", - "type": "timeseries" + "unit": "s" }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 6, + "y": 42 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 45 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Failed Flushes", - "type": "timeseries" + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 45 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Blocks Cleared (1h)", - "type": "timeseries" + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" }, { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 45 + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "title": "Push Latency (Ingester)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 36, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Flush Duration", - "type": "timeseries" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 6, + "y": 47 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 0, - "y": 50 - }, - "id": 72, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))", - "interval": "", - "legendFormat": "accepted", - "refId": "A" - }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))", - "interval": "", - "legendFormat": "refused", - "refId": "B" - } - ], - "title": "OTEL Distributor Spans/second", - "type": "timeseries" + "editorMode": "code", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "range": true, + "refId": "A" }, { "datasource": { + "type": "prometheus", "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 6, - "y": 50 + "editorMode": "code", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "editorMode": "code", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "range": true, + "refId": "C" + } + ], + "title": "Push Latency (Generator)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Push Latency (Ingester)", - "type": "timeseries" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 52 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d452322apre", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 6, - "y": 55 - }, - "id": 108, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "range": true, - "refId": "C" - } - ], - "title": "Push Latency (Generator)", - "type": "timeseries" + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 60 - }, - "id": 109, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d452322apre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))", - "interval": "", - "legendFormat": "accepted", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", - "interval": "", - "legendFormat": "refused {{reason}}", - "range": true, - "refId": "B" - } - ], - "title": "Generator Spans/second", - "type": "timeseries" + "editorMode": "code", + "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)", + "interval": "", + "legendFormat": "refused {{reason}}", + "range": true, + "refId": "B" } ], - "title": "Write", - "type": "row" + "title": "Generator Spans/second", + "type": "timeseries" }, { "collapsed": true, @@ -4267,7 +3638,7 @@ "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 57 }, "id": 28, "panels": [ @@ -4341,8 +3712,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4430,8 +3800,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4497,7 +3866,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 58 }, "id": 56, "panels": [ @@ -4571,8 +3940,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4660,8 +4028,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4756,8 +4123,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4828,7 +4194,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 59 }, "id": 81, "panels": [ @@ -4901,8 +4267,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4990,8 +4355,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5079,8 +4443,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -5170,8 +4533,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -5392,8 +4754,8 @@ { "current": { "selected": false, - "text": "default", - "value": "default" + "text": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_prometheus_0", + "value": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_prometheus_0" }, "hide": 0, "includeAll": false, diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index 76b463c..f148e26 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -36,6 +36,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, + "id": 34, "links": [ { "asDropdown": true, @@ -1562,101 +1563,6 @@ "title": "Bytes Used", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 49 - }, - "id": 127, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(level) (tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Bytes Compacted (by level)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -1718,7 +1624,7 @@ "gridPos": { "h": 8, "w": 6, - "x": 12, + "x": 6, "y": 49 }, "id": 53, @@ -1827,7 +1733,7 @@ "gridPos": { "h": 8, "w": 6, - "x": 18, + "x": 12, "y": 49 }, "id": 35, @@ -2489,17 +2395,17 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "sum(increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", + "expr": "sum by(job) (increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "legendFormat": "compaction_err", "refId": "B" }, { - "expr": "sum(increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", + "expr": "sum by(job) (increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "legendFormat": "retention_err", "refId": "C" }, { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)", + "expr": "sum by(job) (increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "legendFormat": "blocklist_err", "refId": "D" } @@ -2587,7 +2493,7 @@ "targets": [ { "exemplar": true, - "expr": "avg(tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) by (tenant)", + "expr": "avg by(tenant) (tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", "instant": false, "interval": "", "legendFormat": "{{tenant}}", @@ -3138,8 +3044,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3232,8 +3137,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3267,19 +3171,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -3332,8 +3236,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3367,19 +3270,19 @@ "pluginVersion": "9.0.0-d452322apre", "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Pusher/Push.*\"}[5m])))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Pusher/Push.*\"}[5m])))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Pusher/Push.*\"}[5m])))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -3475,8 +3378,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3511,7 +3413,7 @@ "uid": "${prometheusds}" }, "editorMode": "builder", - "expr": "sum(rate(tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)", + "expr": "sum by(level) (rate(tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "interval": "", "legendFormat": "__auto", "range": true, @@ -3566,8 +3468,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3602,7 +3503,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum(increase(tempodb_compaction_blocks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)", + "expr": "sum by(level) (increase(tempodb_compaction_blocks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", "interval": "", "legendFormat": "__auto", "range": true, From 8b9791fa9e0190da4ca7f36cd578abedf36d553a Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Tue, 27 Aug 2024 10:42:16 +0200 Subject: [PATCH 14/14] Remove panels referring to metrics not present --- src/grafana_dashboards/tempo-operational.json | 1361 ++++++----------- src/grafana_dashboards/tempo.json | 233 +-- 2 files changed, 490 insertions(+), 1104 deletions(-) diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json index 1ecbcf1..bfdab69 100644 --- a/src/grafana_dashboards/tempo-operational.json +++ b/src/grafana_dashboards/tempo-operational.json @@ -3629,7 +3629,7 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "datasource": { "type": "prometheus", "uid": "${prometheusds}" @@ -3641,551 +3641,224 @@ "y": 57 }, "id": 28, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 117, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Backend HTTP response codes. Look for increase in 5xx/4xx status codes as well as latency spikes.", - "mode": "markdown" - }, - "pluginVersion": "9.5.3", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 31, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{}[$__rate_interval])) by (status_code, operation)", - "interval": "", - "legendFormat": "{{status_code}}-{{operation}}", - "refId": "A" - } - ], - "title": "Requests/Second", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", - "legendFormat": ".99-{{operation}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", - "legendFormat": ".9-{{operation}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", - "legendFormat": ".5-{{operation}}", - "refId": "C" - } - ], - "title": "Latency By Operation", - "type": "timeseries" - } - ], + "panels": [], "title": "Backend", "type": "row" }, { - "collapsed": true, "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "type": "datasource", + "uid": "grafana" }, "gridPos": { - "h": 1, + "h": 2, "w": 24, "x": 0, "y": 58 }, - "id": 56, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 45 - }, - "id": 118, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "State of Tempo cluster. Look for unhealthy nodes and drops in number of members.", - "mode": "markdown" - }, - "pluginVersion": "9.5.3", - "transparent": true, - "type": "text" + "id": 117, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false }, - { - "datasource": { - "uid": "${prometheusds}" + "content": "Backend HTTP response codes. Look for increase in 5xx/4xx status codes as well as latency spikes.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 47 - }, - "id": 62, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "tempo_memberlist_client_cluster_node_health_score{}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Node Health", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 47 - }, - "id": 63, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + { + "color": "red", + "value": 80 + } + ] }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "min(tempo_memberlist_client_cluster_members_count{})", - "interval": "", - "legendFormat": "min", - "refId": "A" - }, - { - "expr": "max(tempo_memberlist_client_cluster_members_count{})", - "interval": "", - "legendFormat": "max", - "refId": "B" - } - ], - "title": "Member Count", - "type": "timeseries" + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{}[$__rate_interval])) by (status_code, operation)", + "interval": "", + "legendFormat": "{{status_code}}-{{operation}}", + "refId": "A" + } + ], + "title": "Requests/Second", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 47 - }, - "id": 60, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "expr": "min(tempo_memberlist_client_kv_store_count{})", - "interval": "", - "legendFormat": "min", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "expr": "max(tempo_memberlist_client_kv_store_count{})", - "interval": "", - "legendFormat": "max", - "refId": "B" - } - ], - "title": "KV Store Count", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", + "legendFormat": ".99-{{operation}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", + "legendFormat": ".9-{{operation}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))", + "legendFormat": ".5-{{operation}}", + "refId": "C" } ], - "title": "Ring/Memberlist", - "type": "row" + "title": "Latency By Operation", + "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "datasource": { "type": "prometheus", "uid": "${prometheusds}" @@ -4194,393 +3867,325 @@ "h": 1, "w": 24, "x": 0, - "y": 59 + "y": 68 }, - "id": 81, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 119, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "State of periodic compaction. Look for inconsistent gaps pattern.", - "mode": "markdown" - }, - "pluginVersion": "9.5.3", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 48 - }, - "id": 83, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_objects_combined_total{}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Objects Combined / s", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${prometheusds}" + "id": 56, + "panels": [], + "title": "Ring/Memberlist", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 118, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "State of Tempo cluster. Look for unhealthy nodes and drops in number of members.", + "mode": "markdown" + }, + "pluginVersion": "9.5.3", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 48 - }, - "id": 85, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_objects_written_total{}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Objects Written / s", - "type": "timeseries" + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 71 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "expr": "tempo_memberlist_client_cluster_node_health_score{}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Node Health", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 48 - }, - "id": 88, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "builder", - "expr": "sum(rate(tempodb_compaction_bytes_written_total{}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Bytes Written / s", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 71 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "expr": "min(tempo_memberlist_client_cluster_members_count{})", + "interval": "", + "legendFormat": "min", + "refId": "A" }, { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "expr": "max(tempo_memberlist_client_cluster_members_count{})", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Member Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 48 - }, - "id": 86, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "sum(increase(tempodb_compaction_blocks_total{}[5m])) by (level)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Blocks Compacted", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 71 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.0.0-d373beebpre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "expr": "min(tempo_memberlist_client_kv_store_count{})", + "interval": "", + "legendFormat": "min", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "expr": "max(tempo_memberlist_client_kv_store_count{})", + "interval": "", + "legendFormat": "max", + "refId": "B" } ], - "title": "Compactor", - "type": "row" + "title": "KV Store Count", + "type": "timeseries" } ], "refresh": "", diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json index f148e26..5b9ab00 100644 --- a/src/grafana_dashboards/tempo.json +++ b/src/grafana_dashboards/tempo.json @@ -36,7 +36,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 34, + "id": 50, "links": [ { "asDropdown": true, @@ -3044,7 +3044,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3137,7 +3138,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3236,7 +3238,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3290,228 +3293,6 @@ ], "title": "Push Latency (Ingester)", "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 91 - }, - "id": 81, - "panels": [], - "title": "Compactor", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 92 - }, - "id": 331, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Compaction is expected to happen periodically. Blocks should be compacted to higher levels. Look for any irregularities that don't match the patterns observed before.", - "mode": "markdown" - }, - "pluginVersion": "9.5.3", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 94 - }, - "id": 88, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "builder", - "expr": "sum by(level) (rate(tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Bytes Written / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 94 - }, - "id": 86, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.0-d373beebpre", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${prometheusds}" - }, - "editorMode": "code", - "expr": "sum by(level) (increase(tempodb_compaction_blocks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Blocks Compacted", - "type": "timeseries" } ], "refresh": "",