From 649a263790579e004931044538e779b2d40b989b Mon Sep 17 00:00:00 2001 From: Chris Blumentritt Date: Wed, 8 May 2024 10:11:08 -0500 Subject: [PATCH 1/5] Enable servicemonitor in fluentbit helm config (#253) This enables the fluentbit service monitor fluentbit metrics in prometheus Signed-off-by: Chris Blumentritt --- helm-configs/fluentbit/fluentbit-helm-overrides.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helm-configs/fluentbit/fluentbit-helm-overrides.yaml b/helm-configs/fluentbit/fluentbit-helm-overrides.yaml index c345a1df..2eab8cde 100644 --- a/helm-configs/fluentbit/fluentbit-helm-overrides.yaml +++ b/helm-configs/fluentbit/fluentbit-helm-overrides.yaml @@ -24,6 +24,13 @@ daemonSetVolumeMounts: - name: varlogcontainers mountPath: /var/log/containers readOnly: true +serviceMonitor: + enabled: true + namespace: default + interval: 10s + scrapeTimeout: 10s + selector: + application: fluentbit-fluent-bit config: outputs: | [OUTPUT] From 82981bb34c14e02db54d0788270f0c6485884a5e Mon Sep 17 00:00:00 2001 From: Chris Blumentritt Date: Wed, 8 May 2024 10:11:37 -0500 Subject: [PATCH 2/5] Adding fluentbit grafana dashboard (#254) Signed-off-by: Chris Blumentritt --- etc/grafana-dashboards/Fluentbit_Metrics.json | 1483 +++++++++++++++++ 1 file changed, 1483 insertions(+) create mode 100644 etc/grafana-dashboards/Fluentbit_Metrics.json diff --git a/etc/grafana-dashboards/Fluentbit_Metrics.json b/etc/grafana-dashboards/Fluentbit_Metrics.json new file mode 100644 index 00000000..83bab4f1 --- /dev/null +++ b/etc/grafana-dashboards/Fluentbit_Metrics.json @@ -0,0 +1,1483 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.3.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph (old)", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Inspired by https://grafana.com/grafana/dashboards/7752", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7752, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_info{pod=~\".*fluent-bit.*\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active Fluent-bit", + "refId": "A" + } + ], + "title": "Fluent Bits", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 2, + "y": 0 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ready Nodes", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a", + "value": 2 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 4, + "y": 0 + }, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_node_status_condition{condition!=\"Ready\",status=\"true\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Non-Ready Nodes", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 33, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "1 - avg (rate (container_cpu_usage_seconds_total{container=\"fluent-bit\"}[1m]))", + "instant": false, + "refId": "A" + } + ], + "title": "CPU Utilisation", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "1 - sum(container_memory_working_set_bytes{container=\"fluent-bit\"}) / sum(node_memory_MemTotal_bytes)", + "instant": false, + "refId": "A" + } + ], + "title": "Memory Utilisation", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 34, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_resource_requests_cpu_cores) / sum(kube_node_status_allocatable_cpu_cores)", + "instant": false, + "refId": "A" + } + ], + "title": "CPU Requests Commitment", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_resource_requests_memory_bytes) / sum(kube_node_status_allocatable_memory_bytes)", + "instant": false, + "refId": "A" + } + ], + "title": "Memory Requests Commitment", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 35, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores) / sum(kube_node_status_allocatable_cpu_cores)", + "instant": false, + "refId": "A" + } + ], + "title": "CPU Limits Commitment", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 39, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_resource_limits_memory_bytes) / sum(kube_node_status_allocatable_memory_bytes)", + "instant": false, + "refId": "A" + } + ], + "title": "Memory Limits Commitment", + "transparent": true, + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_input_bytes_total[5m])) by (instance, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}/{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input Bytes Processing Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_output_proc_bytes_total[5m])) by (instance, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ pod }}/{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Output Bytes Processing Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_input_records_total[5m])) by (instance, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}/{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input Records Processing Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_output_proc_records_total[5m])) by (instance, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ pod }}/{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Output Record Processing Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_output_retries_total[1m])) by (instance, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} Retries to {{name}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_output_retries_failed_total[1m])) by (instance, name)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} Failed Retries to {{ name }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Output Retry/Failed Rates", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentbit_output_errors_total[1m])) by (instance, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}/{{ name }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Output Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "errors/sec", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Fluent Bit", + "uid": "fluentbit", + "version": 3, + "weekStart": "" +} From 36718b840a5d4d3a0967cbdceae674bcb58d99c1 Mon Sep 17 00:00:00 2001 From: "phillip.toohill" Date: Wed, 8 May 2024 15:10:32 -0500 Subject: [PATCH 3/5] Updating ingress repo and bumping version (#255) --- .../ingress/external/helm/ingress-helm-overrides.yaml | 2 +- kustomize/ingress/external/kustomization.yaml | 8 +++----- .../ingress/grafana/helm/ingress-helm-overrides.yaml | 2 +- kustomize/ingress/grafana/kustomization.yaml | 6 ++---- .../ingress/internal/helm/ingress-helm-overrides.yaml | 2 +- kustomize/ingress/internal/kustomization.yaml | 6 ++---- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/kustomize/ingress/external/helm/ingress-helm-overrides.yaml b/kustomize/ingress/external/helm/ingress-helm-overrides.yaml index 3d50942e..9b5fb159 100644 --- a/kustomize/ingress/external/helm/ingress-helm-overrides.yaml +++ b/kustomize/ingress/external/helm/ingress-helm-overrides.yaml @@ -9,7 +9,7 @@ deployment: images: tags: entrypoint: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 - ingress: registry.k8s.io/ingress-nginx/controller:v1.8.2 + ingress: registry.k8s.io/ingress-nginx/controller:v1.10.1 ingress_module_init: "docker.io/openstackhelm/neutron:2023.1-ubuntu_jammy" ingress_routed_vip: "docker.io/openstackhelm/neutron:2023.1-ubuntu_jammy" error_pages: registry.k8s.io/defaultbackend:1.4 diff --git a/kustomize/ingress/external/kustomization.yaml b/kustomize/ingress/external/kustomization.yaml index 79c55654..42615c0e 100644 --- a/kustomize/ingress/external/kustomization.yaml +++ b/kustomize/ingress/external/kustomization.yaml @@ -1,16 +1,14 @@ resources: - ns-ingress-nginx.yaml -helmGlobals: - chartHome: ../../../submodules/openstack-helm-infra - helmCharts: - - name: ingress + - name: ingress-nginx releaseName: ingress-nginx + repo: https://kubernetes.github.io/ingress-nginx valuesInline: deployment: cluster: class: nginx-cluster valuesFile: helm/ingress-helm-overrides.yaml namespace: ingress-nginx - includeCRDs: true \ No newline at end of file + includeCRDs: true diff --git a/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml b/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml index ce997ebe..567406c1 100644 --- a/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml +++ b/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml @@ -9,7 +9,7 @@ deployment: images: tags: entrypoint: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 - ingress: registry.k8s.io/ingress-nginx/controller:v1.8.2 + ingress: registry.k8s.io/ingress-nginx/controller:v1.10.1 ingress_module_init: "docker.io/openstackhelm/neutron:2023.1-ubuntu_jammy" ingress_routed_vip: "docker.io/openstackhelm/neutron:2023.1-ubuntu_jammy" error_pages: registry.k8s.io/defaultbackend:1.4 diff --git a/kustomize/ingress/grafana/kustomization.yaml b/kustomize/ingress/grafana/kustomization.yaml index 430c7fba..e571c69a 100644 --- a/kustomize/ingress/grafana/kustomization.yaml +++ b/kustomize/ingress/grafana/kustomization.yaml @@ -1,9 +1,7 @@ -helmGlobals: - chartHome: ../../../submodules/openstack-helm-infra - helmCharts: - - name: ingress + - name: ingress-nginx releaseName: ingress-grafana-internal + repo: https://kubernetes.github.io/ingress-nginx valuesInline: deployment: cluster: diff --git a/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml b/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml index 06721d72..94c94b41 100644 --- a/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml +++ b/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml @@ -9,7 +9,7 @@ deployment: images: tags: entrypoint: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 - ingress: registry.k8s.io/ingress-nginx/controller:v1.8.2 + ingress: registry.k8s.io/ingress-nginx/controller:v1.10.1 ingress_module_init: "docker.io/openstackhelm/neutron:2023.1-ubuntu_jammy" ingress_routed_vip: "docker.io/openstackhelm/neutron:2023.1-ubuntu_jammy" error_pages: registry.k8s.io/defaultbackend:1.4 diff --git a/kustomize/ingress/internal/kustomization.yaml b/kustomize/ingress/internal/kustomization.yaml index e02e76f2..e2c2d836 100644 --- a/kustomize/ingress/internal/kustomization.yaml +++ b/kustomize/ingress/internal/kustomization.yaml @@ -1,9 +1,7 @@ -helmGlobals: - chartHome: ../../../submodules/openstack-helm-infra - helmCharts: - - name: ingress + - name: ingress-nginx releaseName: ingress-openstack-internal + repo: https://kubernetes.github.io/ingress-nginx valuesInline: deployment: cluster: From 2c38b4faebcb804421b2beb93e908a853c5c5c98 Mon Sep 17 00:00:00 2001 From: Derek Noble Date: Fri, 3 May 2024 15:25:59 -0500 Subject: [PATCH 4/5] DOC: Added info for new server with user-data and config drives --- docs/openstack-servers.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/openstack-servers.md b/docs/openstack-servers.md index e1fcd14e..ac4fd597 100644 --- a/docs/openstack-servers.md +++ b/docs/openstack-servers.md @@ -53,6 +53,31 @@ openstack server create ``` +#### Creating a Server with User Data + +You can place user data in a local file and pass it through the --user-data parameter at instance creation. + +``` shell +openstack server create --image ubuntu-cloudimage --flavor 1 \ + --user-data mydata.file VM_INSTANCE +``` + +#### Creating a Server with Config drives + +Config drives are special drives that are attached to an instance when it boots. The instance can mount this drive and read files from it to get information that is normally available through the metadata service. + +To enable the config drive for an instance, pass the --config-drive true parameter to the openstack server create command. + +The following example enables the config drive and passes a user data file and two key/value metadata pairs, all of which are accessible from the config drive: + +``` shell +openstack server create --config-drive true --image my-image-name \ + --flavor 1 --key-name mykey --user-data ./my-user-data.txt \ + --property role=webservers --property essential=false MYINSTANCE +``` + +Read more about Openstack Config drives using the [upstream docs](https://docs.openstack.org/nova/latest/admin/config-drive.html). + #### Delete a server ``` shell From c71badd72c98bd24a6f7b8392989376e087895d7 Mon Sep 17 00:00:00 2001 From: Luke Repko Date: Fri, 10 May 2024 14:18:41 -0500 Subject: [PATCH 5/5] Deprecate MaxScale and Tweak Galera (#256) * fix: Remove MaxScale It was introduced to help thwart DBDeadlocks but it did not help resolve that issue. * fix: Make 5 replicas the default Many recovery issues have been observed when losing a node during chaos testing. Moving to a 5 node cluster has improved recovery success rate. * fix: Improve galera performance These changes were proposed by one of our DBAs to help cope with DBDeadlocks. They seem to have helped reduce occurrences. --- docs/infrastructure-mariadb-ops.md | 28 +-- docs/infrastructure-mariadb.md | 22 --- docs/openstack-skyline.md | 2 +- .../cinder/cinder-helm-overrides.yaml | 2 +- .../glance/glance-helm-overrides.yaml | 2 +- .../gnocchi/gnocchi-helm-overrides.yaml | 2 +- helm-configs/heat/heat-helm-overrides.yaml | 2 +- .../horizon/horizon-helm-overrides.yaml | 2 +- .../keystone/keystone-helm-overrides.yaml | 2 +- .../neutron/neutron-helm-overrides.yaml | 2 +- helm-configs/nova/nova-helm-overrides.yaml | 6 +- .../octavia/octavia-helm-overrides.yaml | 2 +- .../placement/placement-helm-overrides.yaml | 2 +- .../mariadb-cluster/aio/kustomization.yaml | 7 - .../mariadb-cluster/base/kustomization.yaml | 1 - .../mariadb-cluster/base/mariadb-galera.yaml | 14 +- .../base/mariadb-maxscale.yaml | 167 ------------------ kustomize/mariadb-operator/kustomization.yaml | 2 - kustomize/octavia/base/octavia-agent.yaml | 2 +- 19 files changed, 22 insertions(+), 247 deletions(-) delete mode 100644 kustomize/mariadb-cluster/base/mariadb-maxscale.yaml diff --git a/docs/infrastructure-mariadb-ops.md b/docs/infrastructure-mariadb-ops.md index afcbd598..def1e049 100644 --- a/docs/infrastructure-mariadb-ops.md +++ b/docs/infrastructure-mariadb-ops.md @@ -7,9 +7,9 @@ Tips and tricks for managing and operating the MariaDB cluster within a Genestac Sometimes an operator may need to connect to the database to troubleshoot things or otherwise make modifications to the databases in place. The following command can be used to connect to the database from a node within the cluster. ``` shell -mysql -h $(kubectl -n openstack get service maxscale-galera -o jsonpath='{.spec.clusterIP}') \ - -p$(kubectl --namespace openstack get secret maxscale -o jsonpath='{.data.password}' | base64 -d) \ - -u maxscale-galera-client +mysql -h $(kubectl -n openstack get service mariadb-galera-primary -o jsonpath='{.spec.clusterIP}') \ + -p$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d) \ + -u root ``` !!! info @@ -92,25 +92,3 @@ for more information. If you have multiple backups available, the operator is able to infer which backup to restore based on the `spec.targetRecoveryTime` field discussed in the operator documentation [here](https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/BACKUP.md#target-recovery-time). - -## Interacting with the MaxScale REST API - -Refer to the API reference for MaxScale [here](https://mariadb.com/kb/en/mariadb-maxscale-23-08-rest-api/). - -!!! info "Example curl request" - - ``` shell - curl -s -u mariadb-operator:$(kubectl get secret -n openstack maxscale -o jsonpath='{.data.password}' | base64 -d) http://maxscale-galera.openstack.svc.cluster.local:8989/v1/ -D - - ``` - ``` shell - HTTP/1.1 200 OK - Connection: close - ETag: "da39a3ee5e6b4b0d3255bfef95601890afd80709" - Last-Modified: Tue, 30 Apr 2024 20:10:22 GMT - Date: Tue, 30 Apr 24 20:44:17 GMT - X-Frame-Options: Deny - X-XSS-Protection: 1 - Referrer-Policy: same-origin - Cache-Control: no-cache - Content-Length: 0 - ``` diff --git a/docs/infrastructure-mariadb.md b/docs/infrastructure-mariadb.md index 45faa2d4..f9fe2b57 100644 --- a/docs/infrastructure-mariadb.md +++ b/docs/infrastructure-mariadb.md @@ -9,12 +9,6 @@ kubectl --namespace openstack \ --type Opaque \ --from-literal=root-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \ --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" - -# MaxScale -kubectl --namespace openstack \ - create secret generic maxscale \ - --type Opaque \ - --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" ``` ## Deploy the mariadb operator @@ -50,19 +44,3 @@ kubectl --namespace openstack apply -k /opt/genestack/kustomize/mariadb-cluster/ ``` shell kubectl --namespace openstack get mariadbs -w ``` - -## MaxScale - -Within the deployment the OpenStack services use MaxScale for loadlancing and greater reliability. While the MaxScale ecosystem is a good one, there are some limitations that you should be aware of. It is recommended that you review the [MaxScale reference documentation](https://mariadb.com/kb/en/mariadb-maxscale-2302-limitations-and-known-issues-within-mariadb-maxscale) for more about all of the known limitations and potential workarounds available. - -``` mermaid -flowchart TD - A[Connection] ---B{MaxScale} - B ---|ro| C[ES-0] - B ---|rw| D[ES-1] ---|sync| E & C - B ---|ro| E[ES-2] -``` - -### MaxScale GUI - -The MaxScale deployment has access to a built in GUI that can be exposed for further debuging and visibility into the performance of the MariDB backend. For more information on accessing the GUI please refer to the MaxScale documentation that can be found [here](https://mariadb.com/resources/blog/getting-started-with-the-mariadb-maxscale-gui). diff --git a/docs/openstack-skyline.md b/docs/openstack-skyline.md index 70f4a095..619696ac 100644 --- a/docs/openstack-skyline.md +++ b/docs/openstack-skyline.md @@ -17,7 +17,7 @@ kubectl --namespace openstack \ --from-literal=service-domain="service" \ --from-literal=service-project="service" \ --from-literal=service-project-domain="service" \ - --from-literal=db-endpoint="maxscale-galera.openstack.svc.cluster.local" \ + --from-literal=db-endpoint="mariadb-galera-primary.openstack.svc.cluster.local" \ --from-literal=db-name="skyline" \ --from-literal=db-username="skyline" \ --from-literal=db-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \ diff --git a/helm-configs/cinder/cinder-helm-overrides.yaml b/helm-configs/cinder/cinder-helm-overrides.yaml index 136fa62c..9d15d034 100644 --- a/helm-configs/cinder/cinder-helm-overrides.yaml +++ b/helm-configs/cinder/cinder-helm-overrides.yaml @@ -1320,7 +1320,7 @@ endpoints: username: cinder password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /cinder diff --git a/helm-configs/glance/glance-helm-overrides.yaml b/helm-configs/glance/glance-helm-overrides.yaml index 6a5ad87c..b3b19a86 100644 --- a/helm-configs/glance/glance-helm-overrides.yaml +++ b/helm-configs/glance/glance-helm-overrides.yaml @@ -589,7 +589,7 @@ endpoints: username: glance password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /glance diff --git a/helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/helm-configs/gnocchi/gnocchi-helm-overrides.yaml index 9110f4e9..db1c37bb 100644 --- a/helm-configs/gnocchi/gnocchi-helm-overrides.yaml +++ b/helm-configs/gnocchi/gnocchi-helm-overrides.yaml @@ -622,7 +622,7 @@ endpoints: username: gnocchi password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /gnocchi diff --git a/helm-configs/heat/heat-helm-overrides.yaml b/helm-configs/heat/heat-helm-overrides.yaml index 138eb76e..b27640c1 100644 --- a/helm-configs/heat/heat-helm-overrides.yaml +++ b/helm-configs/heat/heat-helm-overrides.yaml @@ -859,7 +859,7 @@ endpoints: username: heat password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /heat diff --git a/helm-configs/horizon/horizon-helm-overrides.yaml b/helm-configs/horizon/horizon-helm-overrides.yaml index 4563074f..298f8238 100644 --- a/helm-configs/horizon/horizon-helm-overrides.yaml +++ b/helm-configs/horizon/horizon-helm-overrides.yaml @@ -7242,7 +7242,7 @@ endpoints: username: horizon password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /horizon diff --git a/helm-configs/keystone/keystone-helm-overrides.yaml b/helm-configs/keystone/keystone-helm-overrides.yaml index cb6f0481..09667ed0 100644 --- a/helm-configs/keystone/keystone-helm-overrides.yaml +++ b/helm-configs/keystone/keystone-helm-overrides.yaml @@ -972,7 +972,7 @@ endpoints: username: keystone password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /keystone diff --git a/helm-configs/neutron/neutron-helm-overrides.yaml b/helm-configs/neutron/neutron-helm-overrides.yaml index 2e9dbdd0..ac8f036f 100644 --- a/helm-configs/neutron/neutron-helm-overrides.yaml +++ b/helm-configs/neutron/neutron-helm-overrides.yaml @@ -2199,7 +2199,7 @@ endpoints: username: neutron password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /neutron diff --git a/helm-configs/nova/nova-helm-overrides.yaml b/helm-configs/nova/nova-helm-overrides.yaml index 75e07d39..450a8a9d 100644 --- a/helm-configs/nova/nova-helm-overrides.yaml +++ b/helm-configs/nova/nova-helm-overrides.yaml @@ -1640,7 +1640,7 @@ endpoints: username: nova password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /nova @@ -1657,7 +1657,7 @@ endpoints: username: nova password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /nova_api @@ -1674,7 +1674,7 @@ endpoints: username: nova password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /nova_cell0 diff --git a/helm-configs/octavia/octavia-helm-overrides.yaml b/helm-configs/octavia/octavia-helm-overrides.yaml index 2865d4c9..1a30a9e2 100644 --- a/helm-configs/octavia/octavia-helm-overrides.yaml +++ b/helm-configs/octavia/octavia-helm-overrides.yaml @@ -466,7 +466,7 @@ endpoints: username: octavia password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /octavia diff --git a/helm-configs/placement/placement-helm-overrides.yaml b/helm-configs/placement/placement-helm-overrides.yaml index f6a2cc8c..9d85dd6e 100644 --- a/helm-configs/placement/placement-helm-overrides.yaml +++ b/helm-configs/placement/placement-helm-overrides.yaml @@ -206,7 +206,7 @@ endpoints: username: nova password: password hosts: - default: maxscale-galera + default: mariadb-galera-primary host_fqdn_override: default: null path: /placement diff --git a/kustomize/mariadb-cluster/aio/kustomization.yaml b/kustomize/mariadb-cluster/aio/kustomization.yaml index 071d1a5d..13943e97 100644 --- a/kustomize/mariadb-cluster/aio/kustomization.yaml +++ b/kustomize/mariadb-cluster/aio/kustomization.yaml @@ -12,10 +12,3 @@ patches: - op: replace path: /spec/replicas value: 2 - - target: - kind: MaxScale - name: maxscale-galera - patch: |- - - op: replace - path: /spec/replicas - value: 1 diff --git a/kustomize/mariadb-cluster/base/kustomization.yaml b/kustomize/mariadb-cluster/base/kustomization.yaml index c074ce60..f297b151 100644 --- a/kustomize/mariadb-cluster/base/kustomization.yaml +++ b/kustomize/mariadb-cluster/base/kustomization.yaml @@ -1,5 +1,4 @@ resources: - mariadb-configmap.yaml - - mariadb-maxscale.yaml - mariadb-galera.yaml - mariadb-backup.yaml diff --git a/kustomize/mariadb-cluster/base/mariadb-galera.yaml b/kustomize/mariadb-cluster/base/mariadb-galera.yaml index bd31f2ec..6479a49d 100644 --- a/kustomize/mariadb-cluster/base/mariadb-galera.yaml +++ b/kustomize/mariadb-cluster/base/mariadb-galera.yaml @@ -24,14 +24,10 @@ spec: storage: 10Gi storageClassName: general - replicas: 3 + replicas: 5 podSecurityContext: runAsUser: 0 - # point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale. - maxScaleRef: - name: maxscale-galera - galera: enabled: true primary: @@ -118,7 +114,7 @@ spec: effect: "NoSchedule" podDisruptionBudget: - maxUnavailable: 33% + maxUnavailable: 40% updateStrategy: type: RollingUpdate @@ -138,18 +134,18 @@ spec: performance_schema=ON innodb_log_buffer_size=33554432 wsrep_slave_threads=144 - wsrep_sync_wait=14 + wsrep_sync_wait=0 innodb_flush_log_at_trx_commit=0 ignore-db-dir=lost+found skip-name-resolve - innodb_buffer_pool_size=1024M + innodb_buffer_pool_size=4G innodb_doublewrite=0 innodb_file_format=Barracuda innodb_file_per_table=1 innodb_flush_method=O_DIRECT innodb_io_capacity=500 innodb_locks_unsafe_for_binlog=1 - innodb_log_file_size=128M + innodb_log_file_size=1G innodb_old_blocks_time=1000 innodb_read_io_threads=8 innodb_write_io_threads=8 diff --git a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml deleted file mode 100644 index 0b30755a..00000000 --- a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml +++ /dev/null @@ -1,167 +0,0 @@ -apiVersion: k8s.mariadb.com/v1alpha1 -kind: MaxScale -metadata: - name: maxscale-galera -spec: - replicas: 3 - - mariaDbRef: - name: mariadb-galera - namespace: openstack - - services: - - name: rw-router - router: readwritesplit - params: - transaction_replay: "true" - transaction_replay_attempts: "10" - transaction_replay_timeout: "5s" - max_slave_connections: "255" - max_replication_lag: "3s" - master_accept_reads: "true" - listener: - name: rw-listener - port: 3306 - protocol: MariaDBProtocol - params: - connection_metadata: "tx_isolation=auto" - suspend: false - suspend: false - - name: rconn-master-router - router: readconnroute - params: - router_options: "master" - max_replication_lag: "3s" - master_accept_reads: "true" - listener: - port: 3307 - - name: rconn-slave-router - router: readconnroute - params: - router_options: "slave" - max_replication_lag: "3s" - listener: - port: 3308 - - monitor: - name: mariadb-monitor - module: galeramon - interval: 2s - cooperativeMonitoring: majority_of_all - params: - disable_master_failback: "false" - available_when_donor: "false" - disable_master_role_setting: "false" - suspend: false - - livenessProbe: - failureThreshold: 3 - tcpSocket: - port: 8989 - initialDelaySeconds: 20 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - - readinessProbe: - failureThreshold: 3 - tcpSocket: - port: 8989 - initialDelaySeconds: 20 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - - admin: - port: 8989 - guiEnabled: false - - config: - params: - log_info: "true" - volumeClaimTemplate: - resources: - requests: - storage: 100Mi - accessModes: - - ReadWriteOnce - storageClassName: general - sync: - database: mysql - interval: 5s - timeout: 10s - - auth: - generate: true - adminUsername: mariadb-operator - adminPasswordSecretKeyRef: - name: maxscale - key: password - deleteDefaultAdmin: true - clientUsername: maxscale-galera-client - clientPasswordSecretKeyRef: - name: maxscale - key: password - clientMaxConnections: 1024 - serverUsername: maxscale-galera-server - serverPasswordSecretKeyRef: - name: maxscale - key: password - serverMaxConnections: 1024 - monitorUsername: maxscale-galera-monitor - monitorPasswordSecretKeyRef: - name: maxscale - key: password - monitorMaxConnections: 128 - syncUsername: maxscale-galera-sync - syncPasswordSecretKeyRef: - name: maxscale - key: password - syncMaxConnections: 128 - - securityContext: - allowPrivilegeEscalation: false - - updateStrategy: - type: RollingUpdate - - kubernetesService: - type: LoadBalancer - annotations: - metallb.universe.tf/address-pool: primary - - connection: - secretName: mxs-galera-conn - port: 3306 - - resources: - requests: - memory: 128Mi - - affinity: - enableAntiAffinity: true - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/worker - operator: In - values: - - worker - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/instance - operator: In - values: - - maxscale-galera - topologyKey: kubernetes.io/hostname - - tolerations: - - key: "k8s.mariadb.com/ha" - operator: "Exists" - effect: "NoSchedule" - - podDisruptionBudget: - maxUnavailable: 33% diff --git a/kustomize/mariadb-operator/kustomization.yaml b/kustomize/mariadb-operator/kustomization.yaml index c23068f1..dc9c9148 100644 --- a/kustomize/mariadb-operator/kustomization.yaml +++ b/kustomize/mariadb-operator/kustomization.yaml @@ -22,8 +22,6 @@ helmCharts: - worker metrics: enabled: true - extrArgs: - - '--log-maxscale' affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/kustomize/octavia/base/octavia-agent.yaml b/kustomize/octavia/base/octavia-agent.yaml index 58fb12ad..94ace373 100644 --- a/kustomize/octavia/base/octavia-agent.yaml +++ b/kustomize/octavia/base/octavia-agent.yaml @@ -81,7 +81,7 @@ spec: - name: PATH value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ - name: DEPENDENCY_SERVICE - value: "openstack:maxscale-galera,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server" + value: "openstack:mariadb-galera-primary,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server" - name: DEPENDENCY_JOBS value: "octavia-db-sync,octavia-ks-user,octavia-ks-endpoints" - name: DEPENDENCY_DAEMONSET