Deprecate MaxScale and Tweak Galera (#256)

* fix: Remove MaxScale

It was introduced to help thwart DBDeadlocks but it did not help resolve
that issue.

* fix: Make 5 replicas the default

Many recovery issues have been observed when losing a node during chaos
testing. Moving to a 5 node cluster has improved recovery success rate.

* fix: Improve galera performance

These changes were proposed by one of our DBAs to help cope with
DBDeadlocks. They seem to have helped reduce occurrences.

docs/infrastructure-mariadb-ops.md

@@ -7,9 +7,9 @@ Tips and tricks for managing and operating the MariaDB cluster within a Genestac
 Sometimes an operator may need to connect to the database to troubleshoot things or otherwise make modifications to the databases in place. The following command can be used to connect to the database from a node within the cluster.
 
 ``` shell
-mysql -h $(kubectl -n openstack get service maxscale-galera -o jsonpath='{.spec.clusterIP}') \
-      -p$(kubectl --namespace openstack get secret maxscale -o jsonpath='{.data.password}' | base64 -d) \
-      -u maxscale-galera-client
+mysql -h $(kubectl -n openstack get service mariadb-galera-primary -o jsonpath='{.spec.clusterIP}') \
+      -p$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d) \
+      -u root
 ```
 
 !!! info
@@ -92,25 +92,3 @@ for more information.
     If you have multiple backups available, the operator is able to infer which
     backup to restore based on the `spec.targetRecoveryTime` field discussed
     in the operator documentation [here](https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/BACKUP.md#target-recovery-time).
-
-## Interacting with the MaxScale REST API
-
-Refer to the API reference for MaxScale [here](https://mariadb.com/kb/en/mariadb-maxscale-23-08-rest-api/).
-
-!!! info "Example curl request"
-
-    ``` shell
-    curl -s -u mariadb-operator:$(kubectl get secret -n openstack maxscale -o jsonpath='{.data.password}' | base64 -d) http://maxscale-galera.openstack.svc.cluster.local:8989/v1/ -D -
-    ```
-    ``` shell
-    HTTP/1.1 200 OK
-    Connection: close
-    ETag: "da39a3ee5e6b4b0d3255bfef95601890afd80709"
-    Last-Modified: Tue, 30 Apr 2024 20:10:22 GMT
-    Date: Tue, 30 Apr 24 20:44:17 GMT
-    X-Frame-Options: Deny
-    X-XSS-Protection: 1
-    Referrer-Policy: same-origin
-    Cache-Control: no-cache
-    Content-Length: 0
-    ```

docs/infrastructure-mariadb.md

@@ -9,12 +9,6 @@ kubectl --namespace openstack \
         --type Opaque \
         --from-literal=root-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \
         --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)"
-
-# MaxScale
-kubectl --namespace openstack \
-        create secret generic maxscale \
-        --type Opaque \
-        --from-literal=password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)"
 ```
 
 ## Deploy the mariadb operator
@@ -50,19 +44,3 @@ kubectl --namespace openstack apply -k /opt/genestack/kustomize/mariadb-cluster/
 ``` shell
 kubectl --namespace openstack get mariadbs -w
 ```
-
-## MaxScale
-
-Within the deployment the OpenStack services use MaxScale for loadlancing and greater reliability. While the MaxScale ecosystem is a good one, there are some limitations that you should be aware of. It is recommended that you review the [MaxScale reference documentation](https://mariadb.com/kb/en/mariadb-maxscale-2302-limitations-and-known-issues-within-mariadb-maxscale) for more about all of the known limitations and potential workarounds available.
-
-``` mermaid
-flowchart TD
-    A[Connection] ---B{MaxScale}
-    B ---|ro| C[ES-0]
-    B ---|rw| D[ES-1] ---|sync| E & C
-    B ---|ro| E[ES-2]
-```
-
-### MaxScale GUI
-
-The MaxScale deployment has access to a built in GUI that can be exposed for further debuging and visibility into the performance of the MariDB backend. For more information on accessing the GUI please refer to the MaxScale documentation that can be found [here](https://mariadb.com/resources/blog/getting-started-with-the-mariadb-maxscale-gui).

docs/openstack-skyline.md

@@ -17,7 +17,7 @@ kubectl --namespace openstack \
         --from-literal=service-domain="service" \
         --from-literal=service-project="service" \
         --from-literal=service-project-domain="service" \
-        --from-literal=db-endpoint="maxscale-galera.openstack.svc.cluster.local" \
+        --from-literal=db-endpoint="mariadb-galera-primary.openstack.svc.cluster.local" \
         --from-literal=db-name="skyline" \
         --from-literal=db-username="skyline" \
         --from-literal=db-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \

helm-configs/cinder/cinder-helm-overrides.yaml

@@ -1320,7 +1320,7 @@ endpoints:
         username: cinder
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /cinder

helm-configs/glance/glance-helm-overrides.yaml

@@ -589,7 +589,7 @@ endpoints:
         username: glance
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /glance

helm-configs/gnocchi/gnocchi-helm-overrides.yaml

@@ -622,7 +622,7 @@ endpoints:
         username: gnocchi
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /gnocchi

helm-configs/heat/heat-helm-overrides.yaml

@@ -859,7 +859,7 @@ endpoints:
         username: heat
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /heat

helm-configs/horizon/horizon-helm-overrides.yaml

@@ -7242,7 +7242,7 @@ endpoints:
         username: horizon
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /horizon

helm-configs/keystone/keystone-helm-overrides.yaml

@@ -972,7 +972,7 @@ endpoints:
         username: keystone
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /keystone

helm-configs/neutron/neutron-helm-overrides.yaml

@@ -2199,7 +2199,7 @@ endpoints:
         username: neutron
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /neutron

helm-configs/nova/nova-helm-overrides.yaml

@@ -1640,7 +1640,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /nova
@@ -1657,7 +1657,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /nova_api
@@ -1674,7 +1674,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /nova_cell0

helm-configs/octavia/octavia-helm-overrides.yaml

@@ -466,7 +466,7 @@ endpoints:
         username: octavia
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /octavia

helm-configs/placement/placement-helm-overrides.yaml

@@ -206,7 +206,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: maxscale-galera
+      default: mariadb-galera-primary
     host_fqdn_override:
       default: null
     path: /placement

kustomize/mariadb-cluster/aio/kustomization.yaml

@@ -12,10 +12,3 @@ patches:
       - op: replace
         path: /spec/replicas
         value: 2
-  - target:
-      kind: MaxScale
-      name: maxscale-galera
-    patch: |-
-      - op: replace
-        path: /spec/replicas
-        value: 1

kustomize/mariadb-cluster/base/kustomization.yaml

@@ -1,5 +1,4 @@
 resources:
   - mariadb-configmap.yaml
-  - mariadb-maxscale.yaml
   - mariadb-galera.yaml
   - mariadb-backup.yaml

kustomize/mariadb-cluster/base/mariadb-galera.yaml

@@ -24,14 +24,10 @@ spec:
           storage: 10Gi
       storageClassName: general
 
-  replicas: 3
+  replicas: 5
   podSecurityContext:
     runAsUser: 0
 
-  # point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale.
-  maxScaleRef:
-    name: maxscale-galera
-
   galera:
     enabled: true
     primary:
@@ -118,7 +114,7 @@ spec:
       effect: "NoSchedule"
 
   podDisruptionBudget:
-    maxUnavailable: 33%
+    maxUnavailable: 40%
 
   updateStrategy:
     type: RollingUpdate
@@ -138,18 +134,18 @@ spec:
     performance_schema=ON
     innodb_log_buffer_size=33554432
     wsrep_slave_threads=144
-    wsrep_sync_wait=14
+    wsrep_sync_wait=0
     innodb_flush_log_at_trx_commit=0
     ignore-db-dir=lost+found
     skip-name-resolve
-    innodb_buffer_pool_size=1024M
+    innodb_buffer_pool_size=4G
     innodb_doublewrite=0
     innodb_file_format=Barracuda
     innodb_file_per_table=1
     innodb_flush_method=O_DIRECT
     innodb_io_capacity=500
     innodb_locks_unsafe_for_binlog=1
-    innodb_log_file_size=128M
+    innodb_log_file_size=1G
     innodb_old_blocks_time=1000
     innodb_read_io_threads=8
     innodb_write_io_threads=8