diff --git a/src/grafana_dashboards/tempo-alertmanager.json b/src/grafana_dashboards/tempo-alertmanager.json
deleted file mode 100644
index 6847649..0000000
--- a/src/grafana_dashboards/tempo-alertmanager.json
+++ /dev/null
@@ -1,3519 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 31,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Headlines",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 8,
- "x": 0,
- "y": 1
- },
- "id": 1,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(cortex_alertmanager_alerts{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Total alerts",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 8,
- "x": 8,
- "y": 1
- },
- "id": 2,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(cortex_alertmanager_silences{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Total silences",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 8,
- "x": 16,
- "y": 1
- },
- "id": 3,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "max(cortex_alertmanager_tenants_discovered{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Tenants",
- "type": "stat"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "id": 32,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Alertmanager Distributor",
- "type": "row"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "1000 * sum(rate(cortex_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval])) / sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 12
- },
- "id": 33,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Alerts received",
- "type": "row"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 13
- },
- "hiddenSeries": false,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_alerts_received_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_alerts_invalid_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_alerts_invalid_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "APS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 20
- },
- "id": 34,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Alerts grouping",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 21
- },
- "hiddenSeries": false,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "cortex_alertmanager_dispatcher_aggregation_groups{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{juju_unit}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "per unit Active Aggregation Groups",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 28
- },
- "id": 35,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Alert notifications",
- "type": "row"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_notifications_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "NPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "(sum by(integration) (rate(cortex_alertmanager_notifications_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum by(integration) (rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) > 0 or on() vector(0)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "success - {{ integration }}",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(integration) (rate(cortex_alertmanager_notifications_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed - {{ integration }}",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "NPS by integration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_alertmanager_notification_latency_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_alertmanager_notification_latency_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1000 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 36
- },
- "id": 36,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Alertmanager Configuration Object Store (Alertmanager accesses)",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 44
- },
- "id": 37,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{component=\"alertmanager-storage\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 52
- },
- "id": 38,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Replication",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "max by(juju_unit) (cortex_alertmanager_tenants_owned{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{juju_unit}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Per unit tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(juju_unit) (cortex_alertmanager_alerts{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{juju_unit}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Per unit alerts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(juju_unit) (cortex_alertmanager_silences{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{juju_unit}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Per unit silences",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 60
- },
- "id": 39,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Tenant configuration sync",
- "type": "row"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 61
- },
- "hiddenSeries": false,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_sync_configs_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_sync_configs_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Syncs/sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 61
- },
- "hiddenSeries": false,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{reason}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Syncs/sec (by reason)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 61
- },
- "hiddenSeries": false,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_ring_check_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "errors",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Ring check errors/sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 68
- },
- "id": 40,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Sharding initial state sync",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 69
- },
- "hiddenSeries": false,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "intervalFactor": 2,
- "legendFormat": "{{outcome}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Initial syncs /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 69
- },
- "hiddenSeries": false,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "interval": "1m",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "interval": "1m",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1000 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Initial sync duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 69
- },
- "hiddenSeries": false,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Fetch state from other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 76
- },
- "id": 41,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Sharding runtime state sync",
- "type": "row"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 77
- },
- "hiddenSeries": false,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_replication_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_state_replication_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_replication_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Replicate state to other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 77
- },
- "hiddenSeries": false,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_partial_state_merges_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_partial_state_merges_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_partial_state_merges_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Merge state from other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 77
- },
- "hiddenSeries": false,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_persist_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_alertmanager_state_persist_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Persist state to remote storage /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "refresh": "10s",
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ],
- "value": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ]
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Alertmanager",
- "uid": "b0d38d318bbddd80476246d4930f9e55",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-compactor.json b/src/grafana_dashboards/tempo-compactor.json
deleted file mode 100644
index 32b3015..0000000
--- a/src/grafana_dashboards/tempo-compactor.json
+++ /dev/null
@@ -1,3031 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 23,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Summary",
- "type": "row"
- },
- {
- "aliasColors": {
- "completed": "#7EB26D",
- "failed": "#E24D42",
- "started": "#34CCEB"
- },
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n",
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_runs_started_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "started",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_runs_completed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "completed",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_runs_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Per-instance runs / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\nReset to 0 after the compaction run is completed for all tenants in the shard.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "(\n cortex_compactor_tenants_processing_succeeded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} +\n cortex_compactor_tenants_processing_failed{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} +\n cortex_compactor_tenants_skipped{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}\n)\n/\ncortex_compactor_tenants_discovered{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{juju_unit}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Tenants compaction progress",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Longest time since last successful run\nDisplays the amount of time since the most recent successful execution\nof the compactor.\nThe value shown will be for the compactor replica that has the longest time since its\nlast successful run.\nThe table to the right shows a summary for all compactor replicas.\n\nIf there is no time value, one of the following messages might appear:\n\n- If you see \"No compactor data\" in this panel, that means that no compactors are active yet.\n\n- If you see \"No successful runs\" in this panel, that means that compactors are active, but none\n of them were successfully executed yet.\n\nThese might be expected - for example, if you just recently restarted your compactors,\nthey might not have had a chance to complete their first compaction run.\nHowever, if these messages persist, you should check the health of your compactors.\n\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "mappings": [],
- "noValue": "No compactor data",
- "thresholds": {
- "mode": "absolute",
- "steps": []
- },
- "unit": "s"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Last run"
- },
- "properties": [
- {
- "id": "mappings",
- "value": [
- {
- "options": {
- "from": "-Infinity",
- "result": {
- "color": "text",
- "text": "No successful runs since startup yet"
- },
- "to": 0
- },
- "type": "range"
- }
- ]
- },
- {
- "id": "color",
- "value": {
- "mode": "thresholds"
- }
- },
- {
- "id": "thresholds",
- "value": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "yellow",
- "value": 7200
- },
- {
- "color": "orange",
- "value": 21600
- },
- {
- "color": "red",
- "value": 43200
- }
- ]
- }
- }
- ]
- }
- ]
- },
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 1
- },
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "first"
- ],
- "fields": "/^Last run$/",
- "values": false
- },
- "textMode": "value"
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "max by(juju_unit)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h])\n)\n",
- "format": "table",
- "instant": true,
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "title": "Longest time since last successful run",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transformations": [
- {
- "id": "organize",
- "options": {
- "renameByName": {
- "Value": "Last run",
- "juju_unit": "Compactor"
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "sort": [
- {
- "desc": true,
- "field": "Last run"
- }
- ]
- }
- }
- ],
- "type": "stat",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Last successful run per-compactor replica\nDisplays the compactor replicas, and for each, shows how long it has been since\nits last successful compaction run.\n\nThe value in the status column is based on how long it has been since the last successful compaction.\n\n- Okay: less than 2 hours\n- Delayed: more than 2 hours\n- Late: more than 6 hours\n- Very late: more than 12 hours\n\nIf the status of any compactor replicas are *Late* or *Very late*, check their health.\n\n",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Status"
- },
- "properties": [
- {
- "id": "custom.displayMode",
- "value": "color-background"
- },
- {
- "id": "mappings",
- "value": [
- {
- "options": {
- "from": "-Infinity",
- "result": {
- "color": "transparent",
- "text": "N/A"
- },
- "to": 0
- },
- "type": "range"
- },
- {
- "options": {
- "from": 0,
- "result": {
- "color": "green",
- "text": "Ok"
- },
- "to": 7200
- },
- "type": "range"
- },
- {
- "options": {
- "from": 7200,
- "result": {
- "color": "yellow",
- "text": "Delayed"
- },
- "to": 21600
- },
- "type": "range"
- },
- {
- "options": {
- "from": 21600,
- "result": {
- "color": "orange",
- "text": "Late"
- },
- "to": 43200
- },
- "type": "range"
- },
- {
- "options": {
- "from": 43200,
- "result": {
- "color": "red",
- "text": "Very late"
- },
- "to": "Infinity"
- },
- "type": "range"
- },
- {
- "options": {
- "match": "null+nan",
- "result": {
- "color": "transparent",
- "text": "Unknown"
- }
- },
- "type": "special"
- }
- ]
- },
- {
- "id": "custom.width",
- "value": 86
- },
- {
- "id": "custom.align",
- "value": "center"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Last run"
- },
- "properties": [
- {
- "id": "unit",
- "value": "s"
- },
- {
- "id": "custom.width",
- "value": 74
- },
- {
- "id": "mappings",
- "value": [
- {
- "options": {
- "from": "-Infinity",
- "result": {
- "text": "Never"
- },
- "to": 0
- },
- "type": "range"
- }
- ]
- }
- ]
- }
- ]
- },
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 1
- },
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "max by(juju_unit)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h])\n)\n",
- "format": "table",
- "instant": true,
- "intervalFactor": 2,
- "legendFormat": "Last run",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "title": "Last successful run per-compactor replica",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transformations": [
- {
- "id": "organize",
- "options": {
- "renameByName": {
- "Value": "Last run",
- "juju_unit": "Compactor"
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "sort": [
- {
- "desc": true,
- "field": "Last run"
- }
- ]
- }
- },
- {
- "id": "calculateField",
- "options": {
- "alias": "One",
- "binary": {
- "left": "Last run",
- "operator": "/",
- "right": "Last run"
- },
- "mode": "binary",
- "replaceFields": false
- }
- },
- {
- "id": "calculateField",
- "options": {
- "alias": "Status",
- "binary": {
- "left": "Last run",
- "operator": "*",
- "right": "One"
- },
- "mode": "binary",
- "replaceFields": false
- }
- },
- {
- "id": "filterFieldsByName",
- "options": {
- "include": {
- "names": [
- "Compactor",
- "Last run",
- "Status"
- ]
- }
- }
- }
- ],
- "type": "table",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ]
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 24,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(prometheus_tsdb_compactions_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "compactions",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "TSDB compactions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### TSDB compaction duration\nDisplay the amount of time that it has taken to run a single TSDB compaction.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "TSDB compaction duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 16
- },
- "id": 25,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "avg(max by(user) (cortex_bucket_blocks_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "avg",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Average blocks / tenant",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{user}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Tenants with largest number of blocks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 26,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Garbage collector",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "blocks",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Blocks marked for deletion / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Blocks deletions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 27,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Metadata sync",
- "type": "row"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_meta_syncs_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n-\nsum(rate(cortex_compactor_meta_sync_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Metadata syncs / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Metadata sync duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 40
- },
- "id": 28,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Object Store",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 48
- },
- "id": 29,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 56
- },
- "id": 30,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Key-value store for compactors ring",
- "type": "row"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", kv_name=~\".+\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "refresh": "10s",
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ],
- "value": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ]
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Compactor",
- "uid": "1b3443aea86db629e6efdb7d05c53823",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-config.json b/src/grafana_dashboards/tempo-config.json
deleted file mode 100644
index 381aa17..0000000
--- a/src/grafana_dashboards/tempo-config.json
+++ /dev/null
@@ -1,463 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 3,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Startup config file",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "count by(sha256) (cortex_config_hash{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "sha256:{{sha256}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Startup config file hashes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "instances",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 4,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Runtime config file",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "count by(sha256) (cortex_runtime_config_hash{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "sha256:{{sha256}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Runtime config file hashes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "instances",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "refresh": "10s",
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": [
- "All"
- ],
- "value": [
- "$__all"
- ]
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus datasource",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "queryValue": "",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Config",
- "uid": "5d9d0b4724c0f80d68467088ec61e003",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-operational.json b/src/grafana_dashboards/tempo-operational.json
new file mode 100644
index 0000000..bfdab69
--- /dev/null
+++ b/src/grafana_dashboards/tempo-operational.json
@@ -0,0 +1,4532 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ },
+ {
+ "datasource": {
+ "uid": "$logsds"
+ },
+ "enable": true,
+ "expr": "{diff_{}container='kube-diff-logger'}",
+ "hide": true,
+ "iconColor": "rgba(255, 96, 96, 1)",
+ "name": "diffs",
+ "showIn": 0,
+ "target": {}
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "id": 36,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "tempo"
+ ],
+ "targetBlank": false,
+ "title": "Tempo Dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 38,
+ "panels": [],
+ "title": "General",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 114,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "General operational metrics. CPU metrics require a connected [node_exporter](https://github.com/prometheus/node_exporter), storage panel requires [kubelet metrics](https://kubernetes.io/docs/reference/instrumentation/metrics/) integration.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 0,
+ "y": 3
+ },
+ "id": 24,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "rate(go_gc_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "gcs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 3,
+ "y": 3
+ },
+ "id": 25,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "go_memstats_heap_inuse_bytes{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "go heap",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 6,
+ "y": 3
+ },
+ "id": 23,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "go_goroutines{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "goroutines",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "continuous-BlYlRd"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 9,
+ "y": 3
+ },
+ "id": 42,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "rate(container_cpu_usage_seconds_total{container!=\"POD\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}[$__rate_interval])",
+ "interval": "",
+ "intervalFactor": 5,
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "cpu",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 12,
+ "y": 3
+ },
+ "id": 43,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "container_memory_working_set_bytes{container!=\"POD\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "working set",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 15,
+ "y": 3
+ },
+ "id": 44,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "rate(container_network_receive_bytes_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}[$__rate_interval])",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "rx-{{pod}}",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(container_network_transmit_bytes_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",pod=~\"$component.*\"}[$__rate_interval])",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "tx-{{pod}}",
+ "refId": "B"
+ }
+ ],
+ "title": "tx/rx",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 18,
+ "y": 3
+ },
+ "id": 45,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "kubelet_volume_stats_available_bytes{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",persistentvolumeclaim=~\"$component.*\"}",
+ "legendFormat": "{{persistentvolumeclaim}}",
+ "refId": "A"
+ }
+ ],
+ "title": "data volume free",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 49,
+ "panels": [],
+ "title": "Maintenance",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 115,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Self-maintenance of the cluster. Look for gaps and sudden increases.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 0,
+ "y": 11
+ },
+ "id": 33,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "tempodb_work_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} / tempodb_work_queue_max{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "%age total work queue",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 3,
+ "y": 11
+ },
+ "id": 32,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum by(job) (increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "legendFormat": "compaction_err",
+ "refId": "B"
+ },
+ {
+ "expr": "sum by(job) (increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "legendFormat": "retention_err",
+ "refId": "C"
+ },
+ {
+ "expr": "sum by(job) (increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "legendFormat": "blocklist_err",
+ "refId": "D"
+ }
+ ],
+ "title": "maintenance errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 6,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "always",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 6,
+ "y": 11
+ },
+ "id": 35,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "Blocklist Poll Duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 9,
+ "y": 11
+ },
+ "id": 47,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "avg by(tenant) (tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "{{tenant}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Blocklist Length",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 6,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "always",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 12,
+ "y": 11
+ },
+ "id": 51,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "retention duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 15,
+ "y": 11
+ },
+ "id": 53,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum(increase(tempodb_retention_deleted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "deleted",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "marked_for_deletion",
+ "refId": "B"
+ }
+ ],
+ "title": "retention",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 18,
+ "y": 11
+ },
+ "id": 70,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "increase(kube_pod_container_status_restarts_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]) > 0",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Container Restarts",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "id": 21,
+ "panels": [],
+ "title": "Read",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 120,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Reads by component of Tempo cluster. Look for gaps and sudden increases.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 0,
+ "y": 19
+ },
+ "id": 90,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{status_code}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Queries/Sec (Query Frontend)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 4,
+ "y": 19
+ },
+ "id": 17,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_traces_traceid\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "Query Latency (Query Frontend)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 10,
+ "y": 19
+ },
+ "id": 98,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{status_code}}",
+ "refId": "A",
+ "stepMode": "min"
+ }
+ ],
+ "title": "Search Queries/Sec (Query Frontend)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 14,
+ "y": 19
+ },
+ "id": 94,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A",
+ "stepMode": "min"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B",
+ "stepMode": "min"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\".*api_search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C",
+ "stepMode": "min"
+ }
+ ],
+ "title": "Search Query Latency (Query Frontend)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 0,
+ "y": 24
+ },
+ "id": 91,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{status_code}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Queries/Sec (Querier)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 4,
+ "y": 24
+ },
+ "id": 89,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "Query Latency (Querier)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 10,
+ "y": 24
+ },
+ "id": 99,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{status_code}}",
+ "refId": "A",
+ "stepMode": "min"
+ }
+ ],
+ "title": "Search Queries/Sec (Querier)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 14,
+ "y": 24
+ },
+ "id": 95,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A",
+ "stepMode": "min"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B",
+ "stepMode": "min"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"querier_.*api_search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C",
+ "stepMode": "min"
+ }
+ ],
+ "title": "Search Query Latency (Querier)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 0,
+ "y": 29
+ },
+ "id": 92,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{status_code}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Queries/Sec (Ingester)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 4,
+ "y": 29
+ },
+ "id": 3,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "Query Latency (Ingester)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 10,
+ "y": 29
+ },
+ "id": 100,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "sum by(status_code) (rate(tempo_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{status_code}}",
+ "refId": "A",
+ "stepMode": "min"
+ }
+ ],
+ "title": "Search Queries/Sec (Ingester)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 14,
+ "y": 29
+ },
+ "id": 96,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A",
+ "stepMode": "min"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B",
+ "stepMode": "min"
+ },
+ {
+ "exemplar": true,
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C",
+ "stepMode": "min"
+ }
+ ],
+ "title": "Search Query Latency (Ingester)",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 19,
+ "panels": [],
+ "title": "Write",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 35
+ },
+ "id": 116,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Writes to Tempo backend. Look for gaps in receiving data or sudden spikes in latency.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 0,
+ "y": 37
+ },
+ "id": 10,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Spans Received",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 3,
+ "y": 37
+ },
+ "id": 9,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "builder",
+ "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Traces Created",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 6,
+ "y": 37
+ },
+ "id": 8,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "sum(increase(tempo_ingester_blocks_flushed_total{}[1h]))",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Blocks Flushed (1h)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 9,
+ "y": 37
+ },
+ "id": 12,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "increase(tempo_ingester_failed_flushes_total{}[5m]) > 0",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Failed Flushes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 12,
+ "y": 37
+ },
+ "id": 26,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "sum(increase(tempo_ingester_blocks_cleared_total{}[1h]))",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Blocks Cleared (1h)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 6,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "always",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 15,
+ "y": 37
+ },
+ "id": 36,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "Flush Duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 6,
+ "x": 0,
+ "y": 42
+ },
+ "id": 72,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "accepted",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "refused",
+ "refId": "B"
+ }
+ ],
+ "title": "OTEL Distributor Spans/second",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 7,
+ "x": 6,
+ "y": 42
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
+ }
+ ],
+ "title": "Push Latency (Ingester)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 7,
+ "x": 6,
+ "y": 47
+ },
+ "id": 108,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".9",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.MetricsGenerator/PushSpans'}[$__rate_interval])) by (le))",
+ "interval": "",
+ "legendFormat": ".5",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Push Latency (Generator)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 52
+ },
+ "id": 109,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "accepted",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{}[$__rate_interval])) by (reason)",
+ "interval": "",
+ "legendFormat": "refused {{reason}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Generator Spans/second",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 57
+ },
+ "id": 28,
+ "panels": [],
+ "title": "Backend",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 58
+ },
+ "id": 117,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Backend HTTP response codes. Look for increase in 5xx/4xx status codes as well as latency spikes.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 60
+ },
+ "id": 31,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum(rate(tempodb_backend_request_duration_seconds_count{}[$__rate_interval])) by (status_code, operation)",
+ "interval": "",
+ "legendFormat": "{{status_code}}-{{operation}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Requests/Second",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 60
+ },
+ "id": 30,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))",
+ "legendFormat": ".99-{{operation}}",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))",
+ "legendFormat": ".9-{{operation}}",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{}[$__rate_interval])) by (operation, le))",
+ "legendFormat": ".5-{{operation}}",
+ "refId": "C"
+ }
+ ],
+ "title": "Latency By Operation",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 68
+ },
+ "id": 56,
+ "panels": [],
+ "title": "Ring/Memberlist",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 69
+ },
+ "id": 118,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "State of Tempo cluster. Look for unhealthy nodes and drops in number of members.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 6,
+ "x": 0,
+ "y": 71
+ },
+ "id": 62,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "tempo_memberlist_client_cluster_node_health_score{}",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Node Health",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 6,
+ "x": 6,
+ "y": 71
+ },
+ "id": 63,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "min(tempo_memberlist_client_cluster_members_count{})",
+ "interval": "",
+ "legendFormat": "min",
+ "refId": "A"
+ },
+ {
+ "expr": "max(tempo_memberlist_client_cluster_members_count{})",
+ "interval": "",
+ "legendFormat": "max",
+ "refId": "B"
+ }
+ ],
+ "title": "Member Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 6,
+ "x": 12,
+ "y": 71
+ },
+ "id": 60,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "expr": "min(tempo_memberlist_client_kv_store_count{})",
+ "interval": "",
+ "legendFormat": "min",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "expr": "max(tempo_memberlist_client_kv_store_count{})",
+ "interval": "",
+ "legendFormat": "max",
+ "refId": "B"
+ }
+ ],
+ "title": "KV Store Count",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 38,
+ "style": "dark",
+ "tags": [
+ "tempo"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Loki datasource",
+ "multi": true,
+ "name": "lokids",
+ "options": [],
+ "query": "loki",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Prometheus datasource",
+ "multi": true,
+ "name": "prometheusds",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju unit",
+ "multi": true,
+ "name": "juju_unit",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju application",
+ "multi": true,
+ "name": "juju_application",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju model uuid",
+ "multi": true,
+ "name": "juju_model_uuid",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up,juju_model)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju model",
+ "multi": true,
+ "name": "juju_model",
+ "options": [],
+ "query": {
+ "query": "label_values(up,juju_model)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_prometheus_0",
+ "value": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_prometheus_0"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "ds",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_loki_0",
+ "value": "juju_test-dashboards_3fbf4f3d-3499-4155-8878-d2b3a9264ddc_loki_0"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "logsds",
+ "options": [],
+ "query": "loki",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "isNone": true,
+ "selected": false,
+ "text": "None",
+ "value": ""
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(tempo_build_info, cluster)",
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "cluster",
+ "options": [],
+ "query": {
+ "query": "label_values(tempo_build_info, cluster)",
+ "refId": "ops-cortex-cluster-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "isNone": true,
+ "selected": false,
+ "text": "None",
+ "value": ""
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
+ "refId": "ops-cortex-namespace-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "multi": false,
+ "name": "component",
+ "options": [
+ {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ {
+ "selected": false,
+ "text": "compactor",
+ "value": "compactor"
+ },
+ {
+ "selected": false,
+ "text": "distributor",
+ "value": "distributor"
+ },
+ {
+ "selected": false,
+ "text": "ingester",
+ "value": "ingester"
+ },
+ {
+ "selected": false,
+ "text": "metrics-generator",
+ "value": "metrics-generator"
+ },
+ {
+ "selected": false,
+ "text": "query-frontend",
+ "value": "query-frontend"
+ },
+ {
+ "selected": false,
+ "text": "querier",
+ "value": "querier"
+ }
+ ],
+ "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier",
+ "queryValue": "",
+ "skipUrlSync": false,
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Tempo Operational",
+ "uid": "tempo-coordinator-operational",
+ "version": 1,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/src/grafana_dashboards/tempo-overrides.json b/src/grafana_dashboards/tempo-overrides.json
deleted file mode 100644
index 47edc5e..0000000
--- a/src/grafana_dashboards/tempo-overrides.json
+++ /dev/null
@@ -1,489 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 3,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 1
- },
- "id": 1,
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "max by(limit_name) (cortex_limits_defaults{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Defaults",
- "transformations": [
- {
- "id": "labelsToFields",
- "options": {}
- },
- {
- "id": "merge",
- "options": {}
- },
- {
- "id": "merge",
- "options": {}
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": true
- },
- "indexByName": {
- "Value": 1,
- "limit_name": 0
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "fields": {},
- "sort": [
- {
- "field": "limit_name"
- }
- ]
- }
- }
- ],
- "type": "table"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 4,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 9
- },
- "id": 2,
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "max by(user, limit_name) (cortex_limits_overrides{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",user=~\"${tenant_id}\"})",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Per-tenant overrides",
- "transformations": [
- {
- "id": "labelsToFields",
- "options": {
- "mode": "columns",
- "valueLabel": "limit_name"
- }
- },
- {
- "id": "merge",
- "options": {}
- },
- {
- "id": "merge",
- "options": {}
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": true
- },
- "indexByName": {
- "user": 0
- }
- }
- }
- ],
- "type": "table"
- }
- ],
- "refresh": "",
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": true,
- "text": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ],
- "value": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ]
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": ".*",
- "value": ".*"
- },
- "hide": 0,
- "label": "Tenant ID",
- "name": "tenant_id",
- "options": [
- {
- "selected": true,
- "text": ".*",
- "value": ".*"
- }
- ],
- "query": ".*",
- "skipUrlSync": false,
- "type": "textbox"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Overrides",
- "uid": "1e2c358600ac53f09faea133f811b5bb",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-overview.json b/src/grafana_dashboards/tempo-overview.json
deleted file mode 100644
index 7f38671..0000000
--- a/src/grafana_dashboards/tempo-overview.json
+++ /dev/null
@@ -1,2085 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 20,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Tempo cluster health",
- "type": "row"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "description": "",
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 1
- },
- "id": 1,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "The 'Status' panel shows an overview on the cluster health over the time.\nTo investigate failures, see a specific dashboard:\n\n- Writes\n- Reads\n- Rule evaluations\n- Alerting notifications\n- Object storage\n",
- "mode": "markdown"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "transparent": true,
- "type": "text"
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "fillOpacity": 70,
- "lineWidth": 0,
- "spanNulls": false
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#7EB26D",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 0.01
- },
- {
- "color": "#E24D42",
- "value": 0.05
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 6,
- "y": 1
- },
- "id": 2,
- "options": {
- "alignValue": "left",
- "legend": {
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "mergeValues": true,
- "rowHeight": 0.9,
- "showValue": "never",
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "exemplar": false,
- "expr": "(sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((distributor|cortex|tempo|tempo-write.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.*|error\"}[$__rate_interval])) or vector(0)) / sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((distributor|cortex|tempo|tempo-write.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "Writes",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "exemplar": false,
- "expr": "(sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((query-frontend.*|cortex|tempo|tempo-read.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.*\"}[$__rate_interval])) or vector(0)) / sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/((query-frontend.*|cortex|tempo|tempo-read.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "Reads",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "exemplar": false,
- "expr": "((sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) + sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) or vector(0)) / sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "Rule evaluations",
- "range": true,
- "refId": "C"
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "exemplar": false,
- "expr": "(((sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) or vector(0)) + ((sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\",job=~\"($namespace)/((alertmanager|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})) or vector(0))) / (((sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\",job=~\"($namespace)/((ruler|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) or vector(0)) + ((sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\",job=~\"($namespace)/((alertmanager|cortex|tempo|tempo-backend.*))\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})) or vector(0)))",
- "instant": false,
- "legendFormat": "Alerting notifications",
- "range": true,
- "refId": "D"
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "exemplar": false,
- "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "Object storage",
- "range": true,
- "refId": "E"
- }
- ],
- "title": "Status",
- "type": "state-timeline"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 1
- },
- "id": 3,
- "options": {
- "alertInstanceLabelFilter": "juju_application=\"$juju_application\", juju_model=\"$juju_model\",juju_model_uuid=\"$juju_model_uuid\",juju_unit=\"$juju_unit\"",
- "alertName": "Tempo",
- "dashboardAlerts": false,
- "groupBy": [],
- "groupMode": "default",
- "maxItems": 100,
- "sortOrder": 3,
- "stateFilter": {
- "error": true,
- "firing": true,
- "noData": false,
- "normal": false,
- "pending": false
- },
- "viewMode": "list"
- },
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Firing alerts",
- "type": "alertlist"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 21,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Writes",
- "type": "row"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "description": "",
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 9
- },
- "id": 4,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "These panels show an overview on the write path. \nTo examine the write path in detail, see a specific dashboard:\n\n- Writes\n- Writes resources\n- Writes networking\n- Overview resources\n- Overview networking\n",
- "mode": "markdown"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Write requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "1000 * sum(rate(cortex_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval])) / sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Write latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_distributor_received_samples{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "samples / sec",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_distributor_received_exemplars{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "exemplars / sec",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Ingestion / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "cps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 16
- },
- "id": 22,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Reads",
- "type": "row"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "description": "",
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 17
- },
- "id": 8,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "These panels show an overview on the read path. \nTo examine the read path in detail, see a specific dashboard:\n\n- Reads\n- Reads resources\n- Reads networking\n- Overview resources\n- Overview networking\n- Queries\n- Compactor\n",
- "mode": "markdown"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Read requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "1000 * sum(rate(cortex_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])) / sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Read latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "instant queries",
- "color": "#429D48"
- },
- {
- "alias": "range queries",
- "color": "#F1C731"
- },
- {
- "alias": "\"label names\" queries",
- "color": "#2A66CF"
- },
- {
- "alias": "\"label values\" queries",
- "color": "#9E44C1"
- },
- {
- "alias": "series queries",
- "color": "#FFAB57"
- },
- {
- "alias": "remote read queries",
- "color": "#C79424"
- },
- {
- "alias": "metadata queries",
- "color": "#84D586"
- },
- {
- "alias": "exemplar queries",
- "color": "#A1C4FC"
- },
- {
- "alias": "other",
- "color": "#C788DE"
- }
- ],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_query\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "instant queries",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "range queries",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "\"label names\" queries",
- "range": true,
- "refId": "C",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "\"label values\" queries",
- "range": true,
- "refId": "D",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "series queries",
- "range": true,
- "refId": "E",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_read\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "remote read queries",
- "range": true,
- "refId": "F",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_metadata\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "metadata queries",
- "range": true,
- "refId": "G",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"(prometheus|api_prom)_api_v1_query_exemplars\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "exemplar queries",
- "range": true,
- "refId": "H",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route!~\".*(query|query_range|label.*|series|read|metadata|query_exemplars)\",route=~\"(prometheus|api_prom)_api_v1_.*\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "other",
- "range": true,
- "refId": "I",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 23,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Recording and alerting rules",
- "type": "row"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "description": "",
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 25
- },
- "id": 12,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "These panels show an overview on the recording and alerting rules evaluation.\nTo examine the rules evaluation and alerts notifications in detail, see a specific dashboard:\n\n- Ruler\n- Alertmanager\n- Alertmanager resources\n- Overview resources\n- Overview networking\n",
- "mode": "markdown"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "success",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "missed",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rule evaluations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "average",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rule evaluations latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_notifications_sent_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_prometheus_notifications_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_notifications_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Alerting notifications sent to Alertmanager / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 24,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Long-term storage (object storage)",
- "type": "row"
- },
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "description": "",
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 33
- },
- "id": 16,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "These panels show an overview on the long-term storage (object storage).\nTo examine the storage in detail, see a specific dashboard:\n\n- Object store\n- Compactor\n",
- "mode": "markdown"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "attributes",
- "color": "#429D48"
- },
- {
- "alias": "delete",
- "color": "#F1C731"
- },
- {
- "alias": "exists",
- "color": "#2A66CF"
- },
- {
- "alias": "get",
- "color": "#9E44C1"
- },
- {
- "alias": "get_range",
- "color": "#FFAB57"
- },
- {
- "alias": "iter",
- "color": "#C79424"
- },
- {
- "alias": "upload",
- "color": "#84D586"
- }
- ],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "editorMode": "code",
- "expr": "sum(max by(user) (max_over_time(cortex_bucket_blocks_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[15m])))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "blocks",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Total number of blocks in the storage",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "refresh": "10s",
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": true,
- "text": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ],
- "value": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ]
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Overview",
- "uid": "ffcd83628d7d4b5a03d1cafd159e6c9c",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-queries.json b/src/grafana_dashboards/tempo-queries.json
deleted file mode 100644
index 16bad0c..0000000
--- a/src/grafana_dashboards/tempo-queries.json
+++ /dev/null
@@ -1,3986 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 37,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Query-frontend",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queue duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_query_frontend_retries_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Retries",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 1
- },
- "hiddenSeries": false,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (cortex_query_frontend_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{user}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queue length (per user)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 38,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Query-scheduler",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queue duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 9
- },
- "hiddenSeries": false,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (cortex_query_scheduler_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{user}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queue length (per user)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 16
- },
- "id": 39,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Query-frontend - query splitting and results cache",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Intervals per query\nThe average number of split queries (partitioned by time) executed a single input query.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 0,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_frontend_split_queries_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", method=\"split_by_interval_and_results_cache\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "splitting rate",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Intervals per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 6,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_hits{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n/\nsum (\n rate(cortex_cache_fetched_keys{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Hit ratio",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Query results cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 12,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_fetched_keys{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n-\nsum (\n rate(cortex_cache_hits{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Missed query results per second",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Query results cache misses",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Query results cache skipped\nThe % of queries whose results could not be cached.\nIt is tracked for each split query when the splitting by interval is enabled.\n\n",
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 6,
- "x": 18,
- "y": 17
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_frontend_query_result_cache_skipped_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (reason) /\nignoring (reason) group_left sum(rate(cortex_frontend_query_result_cache_attempted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{reason}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Query results cache skipped",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 40,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Query-frontend - query sharding",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Sharded queries ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel only takes into account the type of queries that are supported by query sharding (eg. range queries).\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "sharded queries ratio",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Sharded queries ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Number of sharded queries per query\nThe number of sharded queries that have been executed for a single input query. It only tracks queries that\nhave been successfully rewritten in a shardable way.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 25
- },
- "hiddenSeries": false,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|tempo|tempo-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Number of sharded queries per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 41,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Ingester",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le) (rate(cortex_ingester_queried_series_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_ingester_queried_series_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "1 * sum(rate(cortex_ingester_queried_series_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_ingester_queried_series_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Series per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le) (rate(cortex_ingester_queried_samples_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_ingester_queried_samples_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "1 * sum(rate(cortex_ingester_queried_samples_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_ingester_queried_samples_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Samples per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 33
- },
- "hiddenSeries": false,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le) (rate(cortex_ingester_queried_exemplars_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum by (le) (rate(cortex_ingester_queried_exemplars_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "1 * sum(rate(cortex_ingester_queried_exemplars_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_ingester_queried_exemplars_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Exemplars per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 40
- },
- "id": 42,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Querier",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Number of store-gateways hit per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Refetches of missing blocks per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "Failure Rate": "#E24D42"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 41
- },
- "hiddenSeries": false,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Failure Rate",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Consistency checks failed",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 48
- },
- "id": 43,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "max(cortex_bucket_index_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Max",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "min(cortex_bucket_index_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Min",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "avg(cortex_bucket_index_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Bucket indexes loaded (per querier)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_index_loads_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_index_load_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Bucket indexes load / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 49
- },
- "hiddenSeries": false,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Bucket indexes load latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 56
- },
- "id": 44,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Store-gateway",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "blocks",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Blocks queried / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(data_type) (\n # Exclude \"chunks refetched\".\n rate(cortex_bucket_store_series_data_size_fetched_bytes_sum{component=\"store-gateway\", stage!=\"refetched\", juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{data_type}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Data fetched / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "binBps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 57
- },
- "hiddenSeries": false,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(data_type) (\n # Exclude \"chunks processed\" to only count \"chunks returned\", other than postings and series.\n rate(cortex_bucket_store_series_data_size_touched_bytes_sum{component=\"store-gateway\", stage!=\"processed\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{data_type}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Data touched / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "binBps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 64
- },
- "id": 45,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 65
- },
- "hiddenSeries": false,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{stage}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Series request average latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 65
- },
- "hiddenSeries": false,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(cortex_bucket_store_series_request_stage_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{stage}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Series request 99th percentile latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Series batch preloading efficiency\nThis panel shows the % of time reduced by preloading, for Series() requests which have been\nsplit to 2+ batches. If a Series() request is served within a single batch, then preloading\nis not triggered, and thus not counted in this measurement.\n\n",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 65
- },
- "hiddenSeries": false,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "# Clamping min to 0 because if preloading not useful at all, then the actual value we get is\n# slightly negative because of the small overhead introduced by preloading.\nclamp_min(1 - (\n sum(rate(cortex_bucket_store_series_batch_preloading_wait_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n /\n sum(rate(cortex_bucket_store_series_batch_preloading_load_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n), 0)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "% of time reduced by preloading",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Series batch preloading efficiency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 72
- },
- "id": 46,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 73
- },
- "hiddenSeries": false,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "cortex_bucket_store_blocks_loaded{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Blocks currently loaded",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 73
- },
- "hiddenSeries": false,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Blocks loaded / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 73
- },
- "hiddenSeries": false,
- "id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "successful",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Blocks dropped / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 80
- },
- "id": 47,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 81
- },
- "hiddenSeries": false,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "cortex_bucket_store_indexheader_lazy_load_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} - cortex_bucket_store_indexheader_lazy_unload_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Lazy loaded index-headers",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 81
- },
- "hiddenSeries": false,
- "id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Index-header lazy load duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 88
- },
- "id": 48,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 89
- },
- "hiddenSeries": false,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "hit ratio",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Series hash cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 89
- },
- "hiddenSeries": false,
- "id": 35,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(thanos_store_index_cache_hits_total{item_type=\"ExpandedPostings\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum(rate(thanos_store_index_cache_requests_total{item_type=\"ExpandedPostings\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "hit ratio",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "ExpandedPostings cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 89
- },
- "hiddenSeries": false,
- "id": 36,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_cache_memory_hits_total{name=\"chunks-attributes-cache\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n/\nsum(rate(cortex_cache_memory_requests_total{name=\"chunks-attributes-cache\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "hit ratio",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Chunks attributes in-memory cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "refresh": "10s",
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "hide": 0,
- "includeAll": true,
- "label": "Loki datasource",
- "multi": true,
- "name": "lokids",
- "options": [],
- "query": "loki",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus datasource",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Queries",
- "uid": "b3abe8d5c040395cc36615cb4334c92d",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-reads.json b/src/grafana_dashboards/tempo-reads.json
new file mode 100644
index 0000000..9ad724a
--- /dev/null
+++ b/src/grafana_dashboards/tempo-reads.json
@@ -0,0 +1,1612 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 6,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "tempo"
+ ],
+ "targetBlank": false,
+ "title": "Tempo Dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 15,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Query Frontend",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 20,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Query Frontend is responsible for sharding incoming requests for faster processing in parallel (by the queriers). See [Query Frontend](https://grafana.com/docs/tempo/latest/configuration/#query-frontend) for more details on the component. The panels show queries per second and their latency.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 3
+ },
+ "hiddenSeries": false,
+ "id": 3,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 3
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'api_.*'}[$__rate_interval])) by (le,route)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'api_.*'}[$__rate_interval])) by (route)",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 16,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Querier",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 11
+ },
+ "id": 21,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "The Querier is responsible for querying the backends/cache for the traceID. See [Querier](https://grafana.com/docs/tempo/latest/configuration/#querier) for more details on the component. The panels show queries per second and their latency.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'querier_api_.*'}[$__rate_interval])) by (le,route)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'querier_api_.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'querier_api_.*'}[$__rate_interval])) by (route)",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 20
+ },
+ "id": 17,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Querier External Endpoint",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 21
+ },
+ "id": 22,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "External endpoints are used to offload backend search requests. If they aren't enabled, the panels won't show any data. See [Serverless](https://grafana.com/docs/tempo/latest/operations/backend_search/#serverless-environment) for more information on external endpoints.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 7,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_duration_seconds_bucket{}[$__rate_interval])) by (le,endpoint)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_querier_external_endpoint_duration_seconds_sum{}[$__rate_interval])) by (endpoint) * 1e3 / sum(rate(tempo_querier_external_endpoint_duration_seconds_count{}[$__rate_interval])) by (endpoint)",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 30
+ },
+ "id": 18,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Ingester",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 31
+ },
+ "id": 23,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "The ingester is responsible for batching up traces and pushing them to TempoDB. See [Ingester](https://grafana.com/docs/tempo/latest/configuration/#ingester) for more information.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 9,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (le,route)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Querier/.*'}[$__rate_interval])) by (route)",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 19,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Backend",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 24,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Backend metrics show the overall state of Tempo's backend components.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation='GET'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation='GET'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation='GET'}[$__rate_interval])) by ()",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 38,
+ "style": "dark",
+ "tags": [
+ "tempo"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Loki datasource",
+ "multi": true,
+ "name": "lokids",
+ "options": [],
+ "query": "loki",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Prometheus datasource",
+ "multi": true,
+ "name": "prometheusds",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju unit",
+ "multi": true,
+ "name": "juju_unit",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju application",
+ "multi": true,
+ "name": "juju_application",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju model uuid",
+ "multi": true,
+ "name": "juju_model_uuid",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up,juju_model)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju model",
+ "multi": true,
+ "name": "juju_model",
+ "options": [],
+ "query": {
+ "query": "label_values(up,juju_model)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [],
+ "query": "label_values(tempo_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Tempo / Reads",
+ "uid": "tempo-coordinator-reads",
+ "version": 1,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/src/grafana_dashboards/tempo-ruler.json b/src/grafana_dashboards/tempo-ruler.json
deleted file mode 100644
index 6cf76f4..0000000
--- a/src/grafana_dashboards/tempo-ruler.json
+++ /dev/null
@@ -1,2555 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "tempo"
- ],
- "targetBlank": false,
- "title": "Tempo dashboards",
- "type": "dashboards"
- }
- ],
- "liveNow": false,
- "panels": [
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 0
- },
- "id": 31,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Headlines",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 0,
- "y": 1
- },
- "id": 1,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(cortex_ruler_managers_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Active configurations",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 6,
- "y": 1
- },
- "id": 2,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(cortex_prometheus_rule_group_rules{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Total rules",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "description": "### Read from ingesters - QPS\nNote: Even while operating in Remote ruler mode you will still see values for this panel.\n\nThis is because the metrics are inclusive of intermediate services and are showing the requests that ultimately reach the ingesters.\n\nFor a more detailed view of the read path when using remote ruler mode, see the Remote ruler reads dashboard.\n\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "reqps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 12,
- "y": 1
- },
- "id": 3,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Read from ingesters - QPS",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "reqps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 18,
- "y": 1
- },
- "id": 4,
- "links": [],
- "options": {
- "colorMode": "none",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "mean"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "9.2.1",
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "refId": "A"
- }
- ],
- "title": "Write to ingesters - QPS",
- "type": "stat"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "id": 32,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Rule evaluations global",
- "type": "row"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) - sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "success",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "failed",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "missed",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Evaluations per second",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_prometheus_rule_evaluation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "average",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 12
- },
- "id": 33,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Writes (ingesters)",
- "type": "row"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 13
- },
- "hiddenSeries": false,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 13
- },
- "hiddenSeries": false,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1000 / sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 20
- },
- "id": 34,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Reads (ingesters)",
- "type": "row"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 21
- },
- "hiddenSeries": false,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 21
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_ingester_client_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1000 / sum(rate(cortex_ingester_client_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 28
- },
- "id": 35,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Ruler - key-value store for rulers ring",
- "type": "row"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 10,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 29
- },
- "hiddenSeries": false,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_kv_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval])) * 1000 / sum(rate(cortex_kv_request_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",kv_name=~\"ruler\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ms",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 36
- },
- "id": 36,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Ruler - blocks storage",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_querier_storegateway_instances_hit_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Number of store-gateways hit per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le) (rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "range": true,
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.5, sum by(le) (rate(cortex_querier_storegateway_refetches_per_query_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))) * 1",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "range": true,
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "range": true,
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Refetches of missing blocks per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {
- "Failures / sec": "#E24D42"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 37
- },
- "hiddenSeries": false,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Failures / sec",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Consistency checks failed",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 44
- },
- "id": 37,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Notifications",
- "type": "row"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(user) (rate(cortex_prometheus_notifications_sent_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ user }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Delivery errors",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(user) (rate(cortex_prometheus_notifications_queue_capacity{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ user }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queue length",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 45
- },
- "hiddenSeries": false,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (increase(cortex_prometheus_notifications_dropped_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ user }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Dropped",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 52
- },
- "id": 38,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ user }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Missed iterations",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ user }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 53
- },
- "hiddenSeries": false,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ rule_group }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Failures",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Group evaluations",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 53
- },
- "id": 39,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 54
- },
- "hiddenSeries": false,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.2.1",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ user }}",
- "range": true,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": false
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Rule evaluation per user",
- "type": "row"
- }
- ],
- "refresh": false,
- "schemaVersion": 37,
- "style": "dark",
- "tags": [
- "tempo"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "hide": 0,
- "includeAll": true,
- "label": "Loki datasource",
- "multi": true,
- "name": "lokids",
- "options": [],
- "query": "loki",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "hide": 0,
- "includeAll": true,
- "label": "Prometheus datasource",
- "multi": true,
- "name": "prometheusds",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju unit",
- "multi": true,
- "name": "juju_unit",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju application",
- "multi": true,
- "name": "juju_application",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model uuid",
- "multi": true,
- "name": "juju_model_uuid",
- "options": [],
- "query": {
- "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".*",
- "current": {
- "selected": false,
- "text": "All",
- "value": "$__all"
- },
- "datasource": {
- "uid": "${prometheusds}"
- },
- "definition": "label_values(up,juju_model)",
- "hide": 0,
- "includeAll": true,
- "label": "Juju model",
- "multi": true,
- "name": "juju_model",
- "options": [],
- "query": {
- "query": "label_values(up,juju_model)",
- "refId": "StandardVariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "2024-02-16T08:22:27.546Z",
- "to": "2024-02-16T10:22:27.546Z"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Tempo / Ruler",
- "uid": "631e15d5d85afb2ca8e35d62984eeaa0",
- "version": 1,
- "weekStart": ""
-}
diff --git a/src/grafana_dashboards/tempo-object-store.json b/src/grafana_dashboards/tempo-tenants.json
similarity index 60%
rename from src/grafana_dashboards/tempo-object-store.json
rename to src/grafana_dashboards/tempo-tenants.json
index d8da615..62e2b74 100644
--- a/src/grafana_dashboards/tempo-object-store.json
+++ b/src/grafana_dashboards/tempo-tenants.json
@@ -11,12 +11,6 @@
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
- "target": {
- "limit": 100,
- "matchAny": false,
- "tags": [],
- "type": "dashboard"
- },
"type": "dashboard"
}
]
@@ -24,6 +18,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
+ "id": 10,
"links": [
{
"asDropdown": true,
@@ -34,7 +29,7 @@
"tempo"
],
"targetBlank": false,
- "title": "Tempo dashboards",
+ "title": "Tempo Dashboards",
"type": "dashboards"
}
],
@@ -63,26 +58,51 @@
"refId": "A"
}
],
- "title": "Components",
+ "title": "Tenant info",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 16,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Tenant configuration and limits for the specified tenant. ",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"aliasColors": {},
"bars": false,
+ "columns": [],
"dashLength": 10,
"dashes": false,
"datasource": {
"uid": "${prometheusds}"
},
- "fill": 10,
- "fillGradient": 0,
+ "fill": 1,
+ "fontSize": "100%",
"gridPos": {
"h": 7,
- "w": 12,
+ "w": 24,
"x": 0,
- "y": 1
+ "y": 3
},
- "hiddenSeries": false,
"id": 1,
"legend": {
"avg": false,
@@ -94,43 +114,63 @@
"values": false
},
"lines": true,
- "linewidth": 0,
+ "linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
"percentage": false,
- "pluginVersion": "9.2.1",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
+ "showHeader": true,
+ "sort": {
+ "col": 0,
+ "desc": true
+ },
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
+ "styles": [
+ {
+ "alias": "Time",
+ "align": "auto",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "hidden"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "string",
+ "unit": "short"
+ }
+ ],
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{component}}",
- "refId": "A",
- "step": 10
+ "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{})\n) by (limit_name)\n",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "",
+ "refId": "A"
}
],
"thresholds": [],
- "timeRegions": [],
- "title": "RPS / component",
+ "title": "Limits",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
- "type": "graph",
+ "transform": "table",
+ "type": "table-old",
"xaxis": {
"mode": "time",
"show": true,
@@ -138,7 +178,7 @@
},
"yaxes": [
{
- "format": "reqps",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -148,10 +188,58 @@
"logBase": 1,
"show": false
}
+ ]
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 12,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
],
- "yaxis": {
- "align": false
- }
+ "title": "Ingestion",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 11
+ },
+ "id": 17,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Ingestion statistics for the specified tenant. Look for spikes or gaps in ingestion.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
},
{
"aliasColors": {},
@@ -165,9 +253,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 12,
- "x": 12,
- "y": 1
+ "w": 8,
+ "x": 0,
+ "y": 13
},
"hiddenSeries": false,
"id": 2,
@@ -188,33 +276,63 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "limit",
+ "dashes": true,
+ "fill": 0
+ },
+ {
+ "alias": "burst limit",
+ "dashes": true,
+ "fill": 0
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "expr": "sum(rate(tempo_distributor_bytes_received_total{tenant='$tenant'}[$__rate_interval]))",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{component}}",
- "refId": "A",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "received",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_rate_limit_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_rate_limit_bytes'})\n) by (ingestion_rate_limit_bytes)\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "limit",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='ingestion_burst_size_bytes'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='ingestion_burst_size_bytes'})\n) by (ingestion_burst_size_bytes)\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "burst limit",
+ "refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Error rate / component",
+ "title": "Distributor bytes/s",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -225,7 +343,7 @@
},
"yaxes": [
{
- "format": "percentunit",
+ "format": "Bps",
"logBase": 1,
"min": 0,
"show": true
@@ -240,32 +358,6 @@
"align": false
}
},
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 8
- },
- "id": 12,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Operations",
- "type": "row"
- },
{
"aliasColors": {},
"bars": false,
@@ -274,13 +366,13 @@
"datasource": {
"uid": "${prometheusds}"
},
- "fill": 10,
+ "fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 12,
- "x": 0,
- "y": 9
+ "w": 8,
+ "x": 8,
+ "y": 13
},
"hiddenSeries": false,
"id": 3,
@@ -294,40 +386,49 @@
"values": false
},
"lines": true,
- "linewidth": 0,
+ "linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_distributor_spans_received_total{tenant='$tenant'}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "accepted",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
},
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "expr": "sum(rate(tempo_discarded_spans_total{tenant='$tenant'}[$__rate_interval])) by (reason)",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "refId": "A",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "refused {{ reason }}",
+ "refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "RPS / operation",
+ "title": "Distributor spans/s",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -338,7 +439,7 @@
},
"yaxes": [
{
- "format": "reqps",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -365,9 +466,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 12,
- "x": 12,
- "y": 9
+ "w": 8,
+ "x": 16,
+ "y": 13
},
"hiddenSeries": false,
"id": 4,
@@ -388,33 +489,63 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "global limit",
+ "dashes": true,
+ "fill": 0
+ },
+ {
+ "alias": "local limit",
+ "dashes": true,
+ "fill": 0
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
+ },
+ "expr": "max(tempo_ingester_live_traces{tenant='$tenant'})",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "live traces",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
},
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))",
+ "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_global_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_global_traces_per_user'})\n) by (max_global_traces_per_user)\n",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{operation}}",
- "refId": "A",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "global limit",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='max_local_traces_per_user'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='max_local_traces_per_user'})\n) by (max_local_traces_per_user)\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "local limit",
+ "refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Error rate / operation",
+ "title": "Live traces",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -425,7 +556,7 @@
},
"yaxes": [
{
- "format": "percentunit",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -450,7 +581,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 16
+ "y": 20
},
"id": 13,
"panels": [],
@@ -463,8 +594,34 @@
"refId": "A"
}
],
+ "title": "Reads",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 21
+ },
+ "id": 18,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Traces data reads for the specified tenant. Look for spikes or gaps in ingestion.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"aliasColors": {},
"bars": false,
@@ -477,9 +634,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 8,
+ "w": 12,
"x": 0,
- "y": 17
+ "y": 23
},
"hiddenSeries": false,
"id": 5,
@@ -500,7 +657,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -511,44 +668,21 @@
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000",
+ "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='traces'}[$__rate_interval])) by (status)",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "{{ status }}",
+ "refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Op: Get",
+ "title": "Queries/s (ID lookup)",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -559,7 +693,7 @@
},
"yaxes": [
{
- "format": "ms",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -586,9 +720,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 8,
- "x": 8,
- "y": 17
+ "w": 12,
+ "x": 12,
+ "y": 23
},
"hiddenSeries": false,
"id": 6,
@@ -609,7 +743,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -620,44 +754,21 @@
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))) * 1000",
+ "expr": "sum(rate(tempo_query_frontend_queries_total{tenant='$tenant',op='search'}[$__rate_interval])) by (status)",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "{{ status }}",
+ "refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Op: GetRange",
+ "title": "Queries/s (search)",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -668,7 +779,7 @@
},
"yaxes": [
{
- "format": "ms",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -683,6 +794,57 @@
"align": false
}
},
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 30
+ },
+ "id": 14,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Storage",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 31
+ },
+ "id": 19,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Storage data behaviour for the specified tenant.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"aliasColors": {},
"bars": false,
@@ -695,9 +857,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 8,
- "x": 16,
- "y": 17
+ "w": 12,
+ "x": 0,
+ "y": 33
},
"hiddenSeries": false,
"id": 7,
@@ -706,7 +868,7 @@
"current": false,
"max": false,
"min": false,
- "show": true,
+ "show": false,
"total": false,
"values": false
},
@@ -718,7 +880,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -729,44 +891,21 @@
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"exists\"}[$__rate_interval]))",
+ "expr": "avg(tempodb_blocklist_length{tenant='$tenant'})",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "length",
+ "refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Op: Exists",
+ "title": "Blockslist length",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -777,7 +916,7 @@
},
"yaxes": [
{
- "format": "ms",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -792,31 +931,6 @@
"align": false
}
},
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 14,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "type": "row"
- },
{
"aliasColors": {},
"bars": false,
@@ -829,9 +943,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 8,
- "x": 0,
- "y": 25
+ "w": 12,
+ "x": 12,
+ "y": 33
},
"hiddenSeries": false,
"id": 8,
@@ -840,7 +954,7 @@
"current": false,
"max": false,
"min": false,
- "show": true,
+ "show": false,
"total": false,
"values": false
},
@@ -852,7 +966,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -863,44 +977,21 @@
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"attributes\"}[$__rate_interval]))",
+ "expr": "sum(tempodb_compaction_outstanding_blocks{tenant='$tenant'})\n/\ncount(tempo_build_info{})\n",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "blocks",
+ "refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Op: Attributes",
+ "title": "Outstanding compactions",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -911,7 +1002,7 @@
},
"yaxes": [
{
- "format": "ms",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -926,6 +1017,57 @@
"align": false
}
},
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 15,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Metrics generator",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 20,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Tenant metric generator data. Note that for any data to show up, metrics-generator needs to be show up. See [its documentation](https://grafana.com/docs/tempo/latest/metrics-generator/) for more details on the component.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"aliasColors": {},
"bars": false,
@@ -938,9 +1080,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 8,
- "x": 8,
- "y": 25
+ "w": 12,
+ "x": 0,
+ "y": 43
},
"hiddenSeries": false,
"id": 9,
@@ -949,7 +1091,7 @@
"current": false,
"max": false,
"min": false,
- "show": true,
+ "show": false,
"total": false,
"values": false
},
@@ -961,7 +1103,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -972,44 +1114,21 @@
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))) * 1000",
+ "expr": "sum(rate(tempo_metrics_generator_bytes_received_total{tenant='$tenant'}[$__rate_interval]))",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
- },
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "rate",
+ "refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Op: Upload",
+ "title": "Bytes/s",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -1020,7 +1139,7 @@
},
"yaxes": [
{
- "format": "ms",
+ "format": "Bps",
"logBase": 1,
"min": 0,
"show": true
@@ -1047,9 +1166,9 @@
"fillGradient": 0,
"gridPos": {
"h": 7,
- "w": 8,
- "x": 16,
- "y": 25
+ "w": 12,
+ "x": 12,
+ "y": 43
},
"hiddenSeries": false,
"id": 10,
@@ -1070,55 +1189,48 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "9.2.1",
+ "pluginVersion": "9.5.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "limit",
+ "dashes": true,
+ "fill": 0
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "uid": "$prometheusds"
- },
- "expr": "histogram_quantile(0.99, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "histogram_quantile(0.5, sum by(le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))) * 1000",
+ "expr": "sum(tempo_metrics_generator_registry_active_series{tenant='$tenant'})",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "{{ tenant }}",
+ "refId": "A"
},
{
"datasource": {
- "uid": "$prometheusds"
+ "uid": "${prometheusds}"
},
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval])) * 1000 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",operation=\"delete\"}[$__rate_interval]))",
+ "expr": "max(\n max by (cluster, namespace, limit_name) (tempo_limits_overrides{user='$tenant',limit_name='metrics_generator_max_active_series'})\n or max by (cluster, namespace, limit_name) (tempo_limits_defaults{limit_name='metrics_generator_max_active_series'})\n) by (metrics_generator_max_active_series)\n",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "interval": "1m",
+ "legendFormat": "limit",
+ "refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Op: Delete",
+ "title": "Active series",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -1129,7 +1241,7 @@
},
"yaxes": [
{
- "format": "ms",
+ "format": "short",
"logBase": 1,
"min": 0,
"show": true
@@ -1145,8 +1257,8 @@
}
}
],
- "refresh": "10s",
- "schemaVersion": 37,
+ "refresh": "",
+ "schemaVersion": 38,
"style": "dark",
"tags": [
"tempo"
@@ -1155,17 +1267,31 @@
"list": [
{
"current": {
- "selected": true,
- "text": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ],
- "value": [
- "juju_tempo-self_ad905178-0b39-4676-891e-f2d33079ce73_prometheus_0"
- ]
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
},
"hide": 0,
"includeAll": true,
- "label": "Prometheus",
+ "label": "Loki datasource",
+ "multi": true,
+ "name": "lokids",
+ "options": [],
+ "query": "loki",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Prometheus datasource",
"multi": true,
"name": "prometheusds",
"options": [],
@@ -1294,6 +1420,110 @@
"tagsQuery": "",
"type": "query",
"useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [],
+ "query": "label_values(tempo_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": "label_values(tempo_build_info{}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "single-tenant",
+ "value": "single-tenant"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": false,
+ "label": "tenant",
+ "multi": false,
+ "name": "tenant",
+ "options": [],
+ "query": "label_values(tempodb_blocklist_length{}, tenant)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
}
]
},
@@ -1327,8 +1557,8 @@
]
},
"timezone": "utc",
- "title": "Tempo / Object Store",
- "uid": "e1324ee2a434f4158c00a9ee279d3292",
+ "title": "Tempo / Tenants",
+ "uid": "tempo-coordinator-tenants",
"version": 1,
"weekStart": ""
-}
+}
\ No newline at end of file
diff --git a/src/grafana_dashboards/tempo-writes.json b/src/grafana_dashboards/tempo-writes.json
new file mode 100644
index 0000000..0dbd705
--- /dev/null
+++ b/src/grafana_dashboards/tempo-writes.json
@@ -0,0 +1,1328 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 9,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "tempo"
+ ],
+ "targetBlank": false,
+ "title": "Tempo Dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 17,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Distributor",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 22,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Distributors receive spans and forward them to the appropriate ingesters. See [Distributor](https://grafana.com/docs/tempo/latest/configuration/#distributor) for more details on the component.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 3
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_receiver_accepted_spans{}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "accepted",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_receiver_refused_spans{}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "refused",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Spans/Second",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 3
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempo_distributor_push_duration_seconds_bucket{}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_distributor_push_duration_seconds_sum{}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_distributor_push_duration_seconds_count{}[$__rate_interval])) by ()",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 18,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Ingester",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 11
+ },
+ "id": 21,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "The ingester is responsible for batching up traces and pushing them to TempoDB. See [Ingester](https://grafana.com/docs/tempo/latest/configuration/#ingester) for more details on the component.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 7,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempo_request_duration_seconds_sum{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_request_duration_seconds_count{route=~'/tempopb.Pusher/Push.*'}[$__rate_interval])) by ()",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 20
+ },
+ "id": 19,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Backend - Ingester",
+ "type": "row"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 21
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 20,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Backend - Compactor",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 29
+ },
+ "id": 23,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Compactors stream blocks from the storage backend, combine them and write them back. See [Compactor](https://grafana.com/docs/tempo/latest/configuration/#compactor) for more details on the component.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "aliasColors": {
+ "1xx": "#EAB839",
+ "2xx": "#7EB26D",
+ "3xx": "#6ED0E0",
+ "4xx": "#EF843C",
+ "5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
+ "error": "#E24D42",
+ "success": "#7EB26D"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 10,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 15,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval]),\n 'status', '${1}xx', 'status_code', '([0-9])..'),\n 'status', '${1}', 'status_code', '([a-zA-Z]+)'))\n",
+ "format": "time_series",
+ "interval": "1m",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "QPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 16,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.5.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 99th",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{operation=~'(PUT|POST)'}[$__rate_interval])) by (le,)) * 1e3",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} 50th",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{operation=~'(PUT|POST)'}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{operation=~'(PUT|POST)'}[$__rate_interval])) by ()",
+ "format": "time_series",
+ "interval": "1m",
+ "intervalFactor": 2,
+ "legendFormat": "{{route}} Average",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 38,
+ "style": "dark",
+ "tags": [
+ "tempo"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Loki datasource",
+ "multi": true,
+ "name": "lokids",
+ "options": [],
+ "query": "loki",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "Prometheus datasource",
+ "multi": true,
+ "name": "prometheusds",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju unit",
+ "multi": true,
+ "name": "juju_unit",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju application",
+ "multi": true,
+ "name": "juju_application",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju model uuid",
+ "multi": true,
+ "name": "juju_model_uuid",
+ "options": [],
+ "query": {
+ "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "label_values(up,juju_model)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Juju model",
+ "multi": true,
+ "name": "juju_model",
+ "options": [],
+ "query": {
+ "query": "label_values(up,juju_model)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "default",
+ "value": "default"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Data source",
+ "multi": false,
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "cluster",
+ "multi": true,
+ "name": "cluster",
+ "options": [],
+ "query": "label_values(tempo_build_info, cluster)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": true,
+ "label": "namespace",
+ "multi": true,
+ "name": "namespace",
+ "options": [],
+ "query": "label_values(tempo_build_info{}, namespace)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "utc",
+ "title": "Tempo / Writes",
+ "uid": "tempo-coordinator-writes",
+ "version": 1,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/src/grafana_dashboards/tempo.json b/src/grafana_dashboards/tempo.json
index d7cf799..5b9ab00 100644
--- a/src/grafana_dashboards/tempo.json
+++ b/src/grafana_dashboards/tempo.json
@@ -36,7 +36,21 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
- "links": [],
+ "id": 50,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [
+ "tempo"
+ ],
+ "targetBlank": false,
+ "title": "Tempo Dashboards",
+ "type": "dashboards"
+ }
+ ],
"liveNow": false,
"panels": [
{
@@ -52,12 +66,35 @@
"y": 0
},
"id": 312,
- "panels": [
-
- ],
+ "panels": [],
"title": "At a glance",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 324,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "High-level overview of core tracing system metrics: amount of traces, spans.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"datasource": {
"type": "prometheus",
@@ -68,37 +105,6 @@
"color": {
"mode": "palette-classic"
},
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
"mappings": [],
"thresholds": {
"mode": "absolute",
@@ -121,28 +127,34 @@
"h": 5,
"w": 3,
"x": 0,
- "y": 0
+ "y": 3
},
"id": 320,
"options": {
- "reduceOptions": {
- "values": false,
- "calcs": [
- "last"
- ],
- "fields": "/^version$/"
- },
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "/^version$/",
+ "values": false
+ },
+ "textMode": "auto",
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
+ "pluginVersion": "9.5.3",
"targets": [
{
"datasource": {
@@ -171,37 +183,6 @@
"color": {
"mode": "palette-classic"
},
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
"mappings": [],
"thresholds": {
"mode": "absolute",
@@ -224,21 +205,34 @@
"h": 5,
"w": 7,
"x": 3,
- "y": 0
+ "y": 3
},
"id": 321,
"options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
+ "pluginVersion": "9.5.3",
"targets": [
{
"datasource": {
@@ -266,37 +260,6 @@
"color": {
"mode": "palette-classic"
},
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
"mappings": [],
"thresholds": {
"mode": "absolute",
@@ -319,21 +282,34 @@
"h": 5,
"w": 7,
"x": 10,
- "y": 0
+ "y": 3
},
"id": 322,
"options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
+ "pluginVersion": "9.5.3",
"targets": [
{
"datasource": {
@@ -341,7 +317,7 @@
"uid": "${prometheusds}"
},
"editorMode": "builder",
- "expr": "tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
+ "expr": "sum(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
"interval": "",
"legendFormat": "",
"range": true,
@@ -360,37 +336,6 @@
"color": {
"mode": "palette-classic"
},
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
"mappings": [],
"thresholds": {
"mode": "absolute",
@@ -413,21 +358,34 @@
"h": 5,
"w": 7,
"x": 17,
- "y": 0
+ "y": 3
},
"id": 323,
"options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
+ "pluginVersion": "9.5.3",
"targets": [
{
"expr": "tempo_ingester_flush_size_bytes_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
@@ -444,12 +402,37 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 6
+ "y": 8
},
"id": 134,
"title": "General",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 325,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Quick overview of main metrics: CPU usage, traces ingestion, flush duration. Use for verifying if spans are ingested by the system.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"datasource": {
"type": "prometheus",
@@ -511,7 +494,7 @@
"h": 8,
"w": 16,
"x": 0,
- "y": 7
+ "y": 11
},
"id": 132,
"options": {
@@ -604,7 +587,7 @@
"h": 8,
"w": 8,
"x": 16,
- "y": 7
+ "y": 11
},
"id": 130,
"options": {
@@ -627,7 +610,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_ingester_flush_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -639,7 +622,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_ingester_flush_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -651,7 +634,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_ingester_flush_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -725,7 +708,7 @@
"h": 8,
"w": 8,
"x": 0,
- "y": 15
+ "y": 19
},
"id": 136,
"options": {
@@ -747,7 +730,7 @@
"uid": "${prometheusds}"
},
"editorMode": "builder",
- "expr": "rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])",
+ "expr": "sum(rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
"legendFormat": "accepted spans",
"range": true,
"refId": "A"
@@ -758,7 +741,7 @@
"uid": "${prometheusds}"
},
"editorMode": "code",
- "expr": "rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])",
+ "expr": "sum(rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
"hide": false,
"legendFormat": "refused spans",
"range": true,
@@ -829,7 +812,7 @@
"h": 8,
"w": 8,
"x": 8,
- "y": 15
+ "y": 19
},
"id": 117,
"options": {
@@ -851,7 +834,7 @@
"uid": "${prometheusds}"
},
"editorMode": "code",
- "expr": "sum by (juju_unit) (rate(tempo_ingester_flush_duration_seconds_sum{juju_application=~\"$juju_application\", juju_model=~\"$juju_model\", juju_model_uuid=~\"$juju_model_uuid\", juju_unit=~\"$juju_unit\"}[5m]))\n",
+ "expr": "sum by(juju_unit) (rate(tempo_ingester_flush_duration_seconds_sum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
@@ -922,7 +905,7 @@
"h": 8,
"w": 8,
"x": 16,
- "y": 15
+ "y": 19
},
"id": 139,
"options": {
@@ -945,7 +928,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_distributor_push_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -957,7 +940,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_distributor_push_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -968,7 +951,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_distributor_push_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -986,12 +969,37 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 23
+ "y": 27
},
"id": 129,
"title": "Network",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 28
+ },
+ "id": 326,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Overview of main metrics: CPU usage, traces ingestion, flush duration. Use for verifying if spans are ingested by the system.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"datasource": {
"type": "prometheus",
@@ -1054,7 +1062,7 @@
"h": 8,
"w": 18,
"x": 0,
- "y": 24
+ "y": 30
},
"id": 78,
"options": {
@@ -1077,7 +1085,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1089,7 +1097,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1101,7 +1109,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1175,7 +1183,7 @@
"h": 8,
"w": 6,
"x": 18,
- "y": 24
+ "y": 30
},
"id": 119,
"options": {
@@ -1213,12 +1221,37 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 32
+ "y": 38
},
"id": 123,
"title": "Backend",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 39
+ },
+ "id": 327,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Backend: storage usage, compaction. Look for any abnormalities in compacting data and retention.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"datasource": {
"type": "prometheus",
@@ -1280,7 +1313,7 @@
"h": 8,
"w": 18,
"x": 0,
- "y": 33
+ "y": 41
},
"id": 121,
"options": {
@@ -1375,7 +1408,7 @@
"h": 8,
"w": 6,
"x": 18,
- "y": 33
+ "y": 41
},
"id": 51,
"options": {
@@ -1398,7 +1431,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1410,7 +1443,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1422,7 +1455,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_retention_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1497,7 +1530,7 @@
"h": 8,
"w": 6,
"x": 0,
- "y": 41
+ "y": 49
},
"id": 125,
"options": {
@@ -1547,7 +1580,7 @@
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
- "fillOpacity": 0,
+ "fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
@@ -1560,8 +1593,8 @@
"scaleDistribution": {
"type": "linear"
},
- "showPoints": "auto",
- "spanNulls": false,
+ "showPoints": "never",
+ "spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
@@ -1584,7 +1617,7 @@
}
]
},
- "unit": "decbytes"
+ "unit": "short"
},
"overrides": []
},
@@ -1592,23 +1625,24 @@
"h": 8,
"w": 6,
"x": 6,
- "y": 41
+ "y": 49
},
- "id": 127,
+ "id": 53,
"options": {
"legend": {
"calcs": [
- "lastNotNull"
+ "mean"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
- "mode": "single",
- "sort": "none"
+ "mode": "multi",
+ "sort": "desc"
}
},
+ "pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
@@ -1616,13 +1650,26 @@
"uid": "${prometheusds}"
},
"editorMode": "code",
- "expr": "sum by(level) (tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
- "legendFormat": "__auto",
+ "expr": "sum(increase(tempodb_retention_deleted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "deleted",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "marked_for_deletion",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Bytes Compacted (by level)",
+ "title": "Retention",
"type": "timeseries"
},
{
@@ -1641,7 +1688,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
- "drawStyle": "line",
+ "drawStyle": "points",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
@@ -1651,120 +1698,11 @@
},
"lineInterpolation": "linear",
"lineWidth": 1,
- "pointSize": 5,
+ "pointSize": 6,
"scaleDistribution": {
"type": "linear"
},
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 12,
- "y": 41
- },
- "id": 53,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
- },
- "editorMode": "code",
- "expr": "sum(increase(tempodb_retention_deleted_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
- "interval": "",
- "legendFormat": "deleted",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
- },
- "editorMode": "code",
- "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
- "interval": "",
- "legendFormat": "marked_for_deletion",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Retention",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 6,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "always",
+ "showPoints": "always",
"spanNulls": true,
"stacking": {
"group": "A",
@@ -1795,8 +1733,8 @@
"gridPos": {
"h": 8,
"w": 6,
- "x": 18,
- "y": 41
+ "x": 12,
+ "y": 49
},
"id": 35,
"options": {
@@ -1819,7 +1757,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1831,7 +1769,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1842,7 +1780,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "P0899610D3ECDF9EC"
+ "uid": "${prometheusds}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by(le) (rate(tempodb_blocklist_poll_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
@@ -1865,13 +1803,38 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 49
+ "y": 57
},
"id": 38,
"panels": [],
- "title": "CPU / Memory / Goroutines",
+ "title": "Memory / Goroutines",
"type": "row"
},
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 58
+ },
+ "id": 328,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Memory usage and garbage collection from Go perspective.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
{
"datasource": {
"uid": "${prometheusds}"
@@ -1895,7 +1858,6 @@
"tooltip": false,
"viz": false
},
- "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -1934,7 +1896,7 @@
"h": 5,
"w": 6,
"x": 0,
- "y": 50
+ "y": 60
},
"id": 24,
"options": {
@@ -1984,7 +1946,6 @@
"tooltip": false,
"viz": false
},
- "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -2023,7 +1984,7 @@
"h": 5,
"w": 6,
"x": 6,
- "y": 50
+ "y": 60
},
"id": 25,
"options": {
@@ -2073,7 +2034,6 @@
"tooltip": false,
"viz": false
},
- "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -2112,7 +2072,7 @@
"h": 5,
"w": 6,
"x": 12,
- "y": 50
+ "y": 60
},
"id": 23,
"options": {
@@ -2161,7 +2121,6 @@
"tooltip": false,
"viz": false
},
- "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -2200,7 +2159,7 @@
"h": 5,
"w": 6,
"x": 18,
- "y": 50
+ "y": 60
},
"id": 43,
"options": {
@@ -2237,1264 +2196,1107 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 55
+ "y": 65
},
"id": 49,
- "panels": [
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "axisSoftMax": 1,
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 0,
- "y": 2
- },
- "id": 33,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "expr": "tempodb_work_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} / tempodb_work_queue_max{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "% of work queue filled",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 6,
- "y": 2
- },
- "id": 32,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "expr": "sum(increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)",
- "legendFormat": "compaction_err",
- "refId": "B"
- },
- {
- "expr": "sum(increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)",
- "legendFormat": "retention_err",
- "refId": "C"
- },
- {
- "expr": "sum(increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (job)",
- "legendFormat": "blocklist_err",
- "refId": "D"
- }
- ],
- "title": "Maintenance Errors",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 12,
- "y": 2
- },
- "id": 47,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "exemplar": true,
- "expr": "avg(tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) by (tenant)",
- "instant": false,
- "interval": "",
- "legendFormat": "{{tenant}}",
- "refId": "A"
- }
- ],
- "title": "Blocklist Length",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 18,
- "y": 2
- },
- "id": 70,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "expr": "increase(kube_pod_container_status_restarts_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]) > 0",
- "hide": false,
- "interval": "",
- "legendFormat": "{{pod}}",
- "refId": "A"
- }
- ],
- "title": "Container Restarts",
- "type": "timeseries"
- }
- ],
+ "panels": [],
"title": "Maintenance",
"type": "row"
},
{
- "collapsed": true,
"datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
+ "type": "datasource",
+ "uid": "grafana"
},
"gridPos": {
- "h": 1,
+ "h": 2,
"w": 24,
"x": 0,
- "y": 56
+ "y": 66
},
- "id": 19,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 0,
- "y": 4
- },
- "id": 10,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
- "interval": "",
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Spans Received",
- "type": "timeseries"
+ "id": 329,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
+ "content": "Potential maintenance issues. Look for increases in queues length, errors, restarts as they may mean resources are underprovisioned.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 6,
- "y": 4
- },
- "id": 9,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMax": 1,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "editorMode": "builder",
- "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
- "interval": "",
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Traces Created",
- "type": "timeseries"
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
},
- {
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 68
+ },
+ "id": 33,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "tempodb_work_queue_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} / tempodb_work_queue_max{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "% of work queue filled",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 68
+ },
+ "id": 32,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "sum by(job) (increase(tempodb_compaction_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "legendFormat": "compaction_err",
+ "refId": "B"
+ },
+ {
+ "expr": "sum by(job) (increase(tempodb_retention_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "legendFormat": "retention_err",
+ "refId": "C"
+ },
+ {
+ "expr": "sum by(job) (increase(tempodb_blocklist_poll_errors_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "legendFormat": "blocklist_err",
+ "refId": "D"
+ }
+ ],
+ "title": "Maintenance Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 68
+ },
+ "id": 47,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "avg by(tenant) (tempodb_blocklist_length{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "{{tenant}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Blocklist Length",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 68
+ },
+ "id": 70,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d373beebpre",
+ "targets": [
+ {
+ "expr": "increase(kube_pod_container_status_restarts_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]) > 0",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Container Restarts",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 73
+ },
+ "id": 19,
+ "panels": [],
+ "title": "Write",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 74
+ },
+ "id": 330,
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "Writing new spans into storage. Look for gaps or sudden increases in latency.",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.5.3",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 76
+ },
+ "id": 10,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Spans Received",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 6,
+ "y": 76
+ },
+ "id": 9,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
"datasource": {
+ "type": "prometheus",
"uid": "${prometheusds}"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "editorMode": "builder",
+ "expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Traces Created",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 12,
- "y": 4
- },
- "id": 8,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 12,
+ "y": 76
+ },
+ "id": 8,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "sum(increase(tempo_ingester_blocks_flushed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Blocks Flushed (1h)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "expr": "sum(increase(tempo_ingester_blocks_flushed_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))",
- "interval": "",
- "legendFormat": "{{pod}}",
- "refId": "A"
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 18,
+ "y": 76
+ },
+ "id": 26,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "sum(increase(tempo_ingester_blocks_cleared_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Blocks Cleared (1h)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
- ],
- "title": "Blocks Flushed (1h)",
- "type": "timeseries"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 0,
+ "y": 81
+ },
+ "id": 72,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
{
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 6,
- "x": 18,
- "y": 4
- },
- "id": 26,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "expr": "sum(increase(tempo_ingester_blocks_cleared_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[1h]))",
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Blocks Cleared (1h)",
- "type": "timeseries"
+ "expr": "sum(rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "accepted",
+ "refId": "A"
},
{
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 9
- },
- "id": 72,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
+ "expr": "sum(rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "refused",
+ "refId": "B"
+ }
+ ],
+ "title": "OTEL Distributor Spans/second",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "uid": "${prometheusds}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "expr": "sum(rate(tempo_receiver_accepted_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
- "interval": "",
- "legendFormat": "accepted",
- "refId": "A"
- },
- {
- "expr": "sum(rate(tempo_receiver_refused_spans{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m]))",
- "interval": "",
- "legendFormat": "refused",
- "refId": "B"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
- ],
- "title": "OTEL Distributor Spans/second",
- "type": "timeseries"
- },
- {
- "datasource": {
- "uid": "${prometheusds}"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 12,
- "x": 12,
- "y": 9
- },
- "id": 79,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))",
- "interval": "",
- "legendFormat": ".99",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))",
- "interval": "",
- "legendFormat": ".9",
- "refId": "B"
- },
- {
- "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (le))",
- "interval": "",
- "legendFormat": ".5",
- "refId": "C"
- }
- ],
- "title": "Push Latency (Gateway)",
- "type": "timeseries"
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 81
+ },
+ "id": 79,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
{
- "datasource": {
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 5,
- "w": 12,
- "x": 12,
- "y": 14
- },
- "id": 2,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.0-d452322apre",
- "targets": [
- {
- "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))",
- "interval": "",
- "legendFormat": ".99",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))",
- "interval": "",
- "legendFormat": ".9",
- "refId": "B"
- },
- {
- "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\", route=~\"/tempopb.Pusher/Push.*\"}[5m])) by (le))",
- "interval": "",
- "legendFormat": ".5",
- "refId": "C"
- }
- ],
- "title": "Push Latency (Ingester)",
- "type": "timeseries"
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
}
],
- "title": "Write",
- "type": "row"
+ "title": "Push Latency (Gateway)",
+ "type": "timeseries"
},
{
- "collapsed": true,
"datasource": {
- "type": "prometheus",
"uid": "${prometheusds}"
},
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 61
- },
- "id": 81,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 6,
- "x": 0,
- "y": 61
- },
- "id": 88,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "editorMode": "builder",
- "expr": "sum(rate(tempodb_compaction_bytes_written_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)",
- "interval": "",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Bytes Written / s",
- "type": "timeseries"
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 86
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.0-d452322apre",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Pusher/Push.*\"}[5m])))",
+ "interval": "",
+ "legendFormat": ".99",
+ "refId": "A"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 6,
- "x": 6,
- "y": 61
- },
- "id": 86,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "9.0.0-d373beebpre",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${prometheusds}"
- },
- "editorMode": "code",
- "expr": "sum(increase(tempodb_compaction_blocks_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[5m])) by (level)",
- "interval": "",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Blocks Compacted",
- "type": "timeseries"
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Pusher/Push.*\"}[5m])))",
+ "interval": "",
+ "legendFormat": ".9",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(tempo_request_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",route=~\"/tempopb.Pusher/Push.*\"}[5m])))",
+ "interval": "",
+ "legendFormat": ".5",
+ "refId": "C"
}
],
- "title": "Compactor",
- "type": "row"
+ "title": "Push Latency (Ingester)",
+ "type": "timeseries"
}
],
- "refresh": false,
- "schemaVersion": 37,
+ "refresh": "",
+ "schemaVersion": 38,
"style": "dark",
"tags": [
"tempo"
@@ -3700,7 +3502,7 @@
},
"timezone": "",
"title": "Tempo Overview",
- "uid": "a6175b9cc7ec20591890117c39580030",
+ "uid": "tempo-coordinator-overview",
"version": 1,
"weekStart": ""
}
\ No newline at end of file