diff --git a/tools/metrics/grafana/dashboards/raft.json b/tools/metrics/grafana/dashboards/raft.json index 958e467a5de..2db233ce0bc 100644 --- a/tools/metrics/grafana/dashboards/raft.json +++ b/tools/metrics/grafana/dashboards/raft.json @@ -21,6 +21,19 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 56, + "panels": [], + "title": "Overall", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -84,7 +97,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "id": 47, "options": { @@ -156,7 +169,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "id": 50, "options": { @@ -217,7 +230,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "hiddenSeries": false, "id": 23, @@ -311,6 +324,100 @@ "align": false } }, + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "editorMode": "code", + "expr": "sum (go_memstats_heap_alloc_bytes{region=\"${region}\", job=\"buildbuddy-app\", namespace=\"raft-dev\"}) by (job, pod_name, namespace)", + "instant": false, + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Heap size", + "type": "timeseries" + }, { "aliasColors": {}, "bars": false, @@ -327,11 +434,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 8 + "x": 0, + "y": 17 }, "hiddenSeries": false, - "id": 2, + "id": 1, "legend": { "avg": false, "current": false, @@ -355,7 +462,7 @@ "renderer": "flot", "seriesOverrides": [ { - "$$hashKey": "object:1687", + "$$hashKey": "object:1858", "alias": "QPS", "yaxis": 2 } @@ -369,13 +476,11 @@ "type": "prometheus", "uid": "vm" }, - "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (le)\n)", + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Read\"}[${window}])) by (le)\n)", "interval": "", "legendFormat": "P99", "queryType": "randomWalk", - "range": true, "refId": "A" }, { @@ -384,7 +489,7 @@ "uid": "vm" }, "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (le)\n)", + "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Read\"}[${window}])) by (le)\n)", "interval": "", "legendFormat": "P95", "refId": "B" @@ -395,7 +500,7 @@ "uid": "vm" }, "exemplar": true, - "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (le)\n)", + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Read\"}[${window}])) by (le)\n)", "interval": "", "legendFormat": "P50", "refId": "C" @@ -406,7 +511,7 @@ "uid": "vm" }, "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (grpc_service)", + "expr": "sum(rate(grpc_server_handled_total{region=\"${region}\", grpc_service=\"google.bytestream.ByteStream\", namespace=\"raft-dev\", grpc_method=\"Read\"}[${window}])) by (grpc_service)", "interval": "", "legendFormat": "QPS", "refId": "D" @@ -414,7 +519,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "/Write", + "title": "/Read", "tooltip": { "shared": true, "sort": 0, @@ -428,13 +533,13 @@ }, "yaxes": [ { - "$$hashKey": "object:1694", + "$$hashKey": "object:1865", "format": "s", "logBase": 1, "show": true }, { - "$$hashKey": "object:1695", + "$$hashKey": "object:1866", "format": "reqps", "logBase": 1, "show": true @@ -460,11 +565,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 17 }, "hiddenSeries": false, - "id": 1, + "id": 2, "legend": { "avg": false, "current": false, @@ -488,7 +593,7 @@ "renderer": "flot", "seriesOverrides": [ { - "$$hashKey": "object:1858", + "$$hashKey": "object:1687", "alias": "QPS", "yaxis": 2 } @@ -502,11 +607,13 @@ "type": "prometheus", "uid": "vm" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Read\"}[${window}])) by (le)\n)", + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (le)\n)", "interval": "", "legendFormat": "P99", "queryType": "randomWalk", + "range": true, "refId": "A" }, { @@ -515,7 +622,7 @@ "uid": "vm" }, "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Read\"}[${window}])) by (le)\n)", + "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (le)\n)", "interval": "", "legendFormat": "P95", "refId": "B" @@ -526,7 +633,7 @@ "uid": "vm" }, "exemplar": true, - "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Read\"}[${window}])) by (le)\n)", + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (le)\n)", "interval": "", "legendFormat": "P50", "refId": "C" @@ -537,7 +644,7 @@ "uid": "vm" }, "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{region=\"${region}\", grpc_service=\"google.bytestream.ByteStream\", namespace=\"raft-dev\", grpc_method=\"Read\"}[${window}])) by (grpc_service)", + "expr": "sum(rate(grpc_server_handled_total{region=\"${region}\", namespace=\"raft-dev\", grpc_service=\"google.bytestream.ByteStream\", grpc_method=\"Write\"}[${window}])) by (grpc_service)", "interval": "", "legendFormat": "QPS", "refId": "D" @@ -545,7 +652,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "/Read", + "title": "/Write", "tooltip": { "shared": true, "sort": 0, @@ -559,13 +666,13 @@ }, "yaxes": [ { - "$$hashKey": "object:1865", + "$$hashKey": "object:1694", "format": "s", "logBase": 1, "show": true }, { - "$$hashKey": "object:1866", + "$$hashKey": "object:1695", "format": "reqps", "logBase": 1, "show": true @@ -576,11 +683,11 @@ } }, { + "collapsed": true, "datasource": { "type": "prometheus", "uid": "vm" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -617,6 +724,7 @@ "mode": "off" } }, + "decimals": 3, "mappings": [], "thresholds": { "mode": "absolute", @@ -630,17 +738,17 @@ } ] }, - "unit": "µs" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 25 }, - "id": 22, + "id": 8, "options": { "legend": { "calcs": [], @@ -659,18 +767,18 @@ "type": "prometheus", "uid": "vm" }, - "editorMode": "code", - "expr": "histogram_quantile(${quantile}, sum(rate(buildbuddy_raft_replica_update_duration_usec_bucket{region=\"${region}\"}[${window}])) by (le))", - "instant": false, - "legendFormat": "__auto", - "range": true, + "exemplar": true, + "expr": "sum(rate(buildbuddy_raft_rangecache_lookups{rangecache_event_type=\"hit\", region=\"${region}\"}[${window}]))/sum(rate(buildbuddy_raft_rangecache_lookups{region=\"${region}\"}[${window}]))", + "interval": "", + "legendFormat": "", "refId": "A" } ], - "title": "replica.Update latency (q=${quantile})", + "title": "RangeCache Hit Rate", "type": "timeseries" }, { + "collapsed": true, "datasource": { "type": "prometheus", "uid": "vm" @@ -724,30 +832,26 @@ } ] }, - "unit": "s" + "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 24 + "x": 12, + "y": 25 }, - "id": 16, + "id": 5, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -759,689 +863,923 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(${quantile}, sum by (le, grpc_service, grpc_method) (rate(grpc_server_handling_seconds_bucket{region=\"${region}\", job=\"buildbuddy-app\", grpc_service=\"raft.service.Api\", namespace=\"raft-dev\"})[$window]))", + "expr": "sum(rate(buildbuddy_raft_proposals{region=\"${region}\"}[${window}])) by (pod_name)", "interval": "", - "legendFormat": "{{grpc_method}}", + "legendFormat": "{{pod_name}}", "range": true, "refId": "A" } ], - "title": "gRPC server handling duration, q=${quantile}", + "title": "Raft Proposals", "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 51, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "h": 1, + "w": 24, + "x": 0, + "y": 33 }, - "targets": [ + "id": 58, + "panels": [ { "datasource": { "type": "prometheus", "uid": "vm" }, - "editorMode": "code", - "expr": "sum by (grpc_service, grpc_method, pod_name) (rate(grpc_server_handled_total{job=\"buildbuddy-app\", grpc_service=\"raft.service.Api\", namespace=\"raft-dev\"}[${window}]))", - "instant": false, - "legendFormat": "{{grpc_service}}.{{grpc_method}} {{pod_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Handled gRPC requests per second by method", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "decimals": 3, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(${quantile}, sum by (le, grpc_service, grpc_method) (rate(grpc_server_handling_seconds_bucket{region=\"${region}\", job=\"buildbuddy-app\", grpc_service=\"raft.service.Api\", namespace=\"raft-dev\"})[$window]))", + "interval": "", + "legendFormat": "{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "gRPC server handling duration, q=${quantile}", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "vm" }, - "exemplar": true, - "expr": "sum(rate(buildbuddy_raft_rangecache_lookups{rangecache_event_type=\"hit\", region=\"${region}\"}[${window}]))/sum(rate(buildbuddy_raft_rangecache_lookups{region=\"${region}\"}[${window}]))", - "interval": "", - "legendFormat": "", - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "editorMode": "code", + "expr": "sum by (grpc_service, grpc_method, pod_name) (rate(grpc_server_handled_total{job=\"buildbuddy-app\", grpc_service=\"raft.service.Api\", namespace=\"raft-dev\"}[${window}]))", + "instant": false, + "legendFormat": "{{grpc_service}}.{{grpc_method}} {{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Handled gRPC requests per second by method", + "type": "timeseries" } ], - "title": "RangeCache Hit Rate", - "type": "timeseries" + "title": "gRPC", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "vm" + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 59, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" + "editorMode": "code", + "expr": "histogram_quantile(${quantile}, sum(rate(buildbuddy_raft_replica_update_duration_usec_bucket{region=\"${region}\"}[${window}])) by (le))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "replica.Update latency (q=${quantile})", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "editorMode": "code", + "expr": "histogram_quantile(${quantile}, sum(rate(buildbuddy_raft_range_lock_duration_msec_bucket{region=\"${region}\"}[${window}])) by (pod_name, le))", + "instant": false, + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "range lock latency (q=${quantile})", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "vm" }, - "editorMode": "code", - "expr": "sum (go_memstats_heap_alloc_bytes{region=\"${region}\", job=\"buildbuddy-app\", namespace=\"raft-dev\"}) by (job, pod_name, namespace)", - "instant": false, - "legendFormat": "{{pod_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Heap size", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "expr": "histogram_quantile(${quantile}, sum(rate(buildbuddy_raft_nodehost_method_usec_bucket{region=\"${region}\", nodehost_method=\"SyncPropose\"}[${window}])) by (pod_name, le))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "nodehost.SyncPropose latency (q=${quantile})", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "vm" }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(buildbuddy_raft_proposals{region=\"${region}\"}[${window}])) by (pod_name)", - "interval": "", - "legendFormat": "{{pod_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Raft Proposals", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decmbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "editorMode": "code", + "expr": "sum by (pod_name) (rate(buildbuddy_raft_nodehost_method_usec_count{job=\"buildbuddy-app\",nodehost_method=\"SyncPropose\"}[${window}]))", + "instant": false, + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "nodehost.SyncPropose requests per sec by pod", + "type": "timeseries" } + ], + "title": "Latency", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 }, - "targets": [ + "id": 57, + "panels": [ { + "collapsed": true, "datasource": { "type": "prometheus", "uid": "vm" }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(buildbuddy_raft_bytes{region=\"${region}\"})/1e6", - "interval": "", - "legendFormat": "avg", - "range": true, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "exemplar": true, + "expr": "avg(buildbuddy_raft_ranges{region=\"${region}\"}) by (node_host_id, pod_name)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Raft Ranges", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "vm" }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(buildbuddy_raft_bytes{region=\"${region}\"}/1e6) by (pod_name)", - "hide": false, - "interval": "", - "legendFormat": "{{pod_name}}", - "range": true, - "refId": "B" - } - ], - "title": "Range Sizes", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "expr": "sum(buildbuddy_raft_leases{region=\"${region}\"}) by (node_host_id, pod_name)", + "instant": false, + "legendFormat": "{{pod_name}} - {{node_host_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Lease Count", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { + "collapsed": true, "datasource": { "type": "prometheus", "uid": "vm" }, - "exemplar": true, - "expr": "avg(buildbuddy_raft_ranges{region=\"${region}\"}) by (node_host_id, pod_name)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Raft Ranges", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "vm" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 49, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "vm" - }, - "editorMode": "code", - "expr": "sum(buildbuddy_raft_leases{region=\"${region}\"}) by (node_host_id, pod_name)", - "instant": false, - "legendFormat": "{{pod_name}} - {{node_host_id}}", - "range": true, - "refId": "A" + "editorMode": "code", + "exemplar": true, + "expr": "avg(buildbuddy_raft_bytes{region=\"${region}\"})/1e6", + "interval": "", + "legendFormat": "avg", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "vm" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(buildbuddy_raft_bytes{region=\"${region}\"}/1e6) by (pod_name)", + "hide": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "B" + } + ], + "title": "Range Sizes", + "type": "timeseries" } ], - "title": "Lease Count", - "type": "timeseries" + "title": "Range and Leases", + "type": "row" }, { "collapsed": true, @@ -1449,7 +1787,7 @@ "h": 1, "w": 24, "x": 0, - "y": 56 + "y": 36 }, "id": 55, "panels": [ @@ -1804,7 +2142,7 @@ "h": 1, "w": 24, "x": 0, - "y": 57 + "y": 37 }, "id": 54, "panels": [ @@ -1984,9 +2322,9 @@ "uid": "vm" }, "editorMode": "code", - "expr": "sum(rate(buildbuddy_raft_zombie_cleanup_errors{region=\"${region}\", job=\"buildbuddy-app\"}[${window}])) by (pod_name)", + "expr": "sum(rate(buildbuddy_raft_zombie_cleanup{region=\"${region}\", job=\"buildbuddy-app\"}[${window}])) by (pod_name,status)", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{podname}} - {{status}}", "range": true, "refId": "A" } @@ -2004,7 +2342,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 38 }, "id": 25, "panels": [ @@ -2780,7 +3118,7 @@ "h": 1, "w": 24, "x": 0, - "y": 59 + "y": 39 }, "id": 37, "panels": [ @@ -4002,7 +4340,7 @@ "h": 1, "w": 24, "x": 0, - "y": 60 + "y": 40 }, "id": 20, "panels": [