feat: introduce maxscale

With the release of the mariadb operator v0.25.0 maxscale was introduced to resolve issues with multi-master deployments, enhance scale, and make better use of nodes in the environment. This change creates the maxscale resources and converts our standard deployment systems to use maxscale as the point of ingress. Specific feature we're interested in > point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale. Docs: https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/MAXSCALE.md Related: https://github.com/mariadb-operator/mariadb-operator/releases/tag/v0.0.26 Signed-off-by: Kevin Carter <[email protected]>
rackerlabs · Mar 20, 2024 · ce1005c · ce1005c
1 parent 732b7df
commit ce1005c
Show file tree

Hide file tree

Showing 17 changed files with 159 additions and 32 deletions.
diff --git a/docs/infrastructure-mariadb-connect.md b/docs/infrastructure-mariadb-connect.md
@@ -3,7 +3,7 @@
 Sometimes an operator may need to connect to the database to troubleshoot things or otherwise make modifications to the databases in place. The following command can be used to connect to the database from a node within the cluster.
 
 ``` shell
-mysql -h $(kubectl -n openstack get service mariadb-galera-primary -o jsonpath='{.spec.clusterIP}') \
+mysql -h $(kubectl -n openstack get service maxscale-galera -o jsonpath='{.spec.clusterIP}') \
       -p$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d) \
       -u root
 ```

diff --git a/docs/openstack-skyline.md b/docs/openstack-skyline.md
@@ -17,7 +17,7 @@ kubectl --namespace openstack \
         --from-literal=service-domain="service" \
         --from-literal=service-project="service" \
         --from-literal=service-project-domain="service" \
-        --from-literal=db-endpoint="mariadb-galera-primary.openstack.svc.cluster.local" \
+        --from-literal=db-endpoint="maxscale-galera.openstack.svc.cluster.local" \
         --from-literal=db-name="skyline" \
         --from-literal=db-username="skyline" \
         --from-literal=db-password="$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c${1:-32};echo;)" \

diff --git a/helm-configs/cinder/cinder-helm-overrides.yaml b/helm-configs/cinder/cinder-helm-overrides.yaml
@@ -1320,7 +1320,7 @@ endpoints:
         username: cinder
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /cinder

diff --git a/helm-configs/glance/glance-helm-overrides.yaml b/helm-configs/glance/glance-helm-overrides.yaml
@@ -589,7 +589,7 @@ endpoints:
         username: glance
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /glance

diff --git a/helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/helm-configs/gnocchi/gnocchi-helm-overrides.yaml
@@ -622,7 +622,7 @@ endpoints:
         username: gnocchi
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /gnocchi

diff --git a/helm-configs/heat/heat-helm-overrides.yaml b/helm-configs/heat/heat-helm-overrides.yaml
@@ -859,7 +859,7 @@ endpoints:
         username: heat
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /heat

diff --git a/helm-configs/horizon/horizon-helm-overrides.yaml b/helm-configs/horizon/horizon-helm-overrides.yaml
@@ -7242,7 +7242,7 @@ endpoints:
         username: horizon
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /horizon

diff --git a/helm-configs/keystone/keystone-helm-overrides.yaml b/helm-configs/keystone/keystone-helm-overrides.yaml
@@ -972,7 +972,7 @@ endpoints:
         username: keystone
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /keystone

diff --git a/helm-configs/neutron/neutron-helm-overrides.yaml b/helm-configs/neutron/neutron-helm-overrides.yaml
@@ -2199,7 +2199,7 @@ endpoints:
         username: neutron
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /neutron

diff --git a/helm-configs/nova/nova-helm-overrides.yaml b/helm-configs/nova/nova-helm-overrides.yaml
@@ -1640,7 +1640,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /nova
@@ -1657,7 +1657,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /nova_api
@@ -1674,7 +1674,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /nova_cell0

diff --git a/helm-configs/octavia/octavia-helm-overrides.yaml b/helm-configs/octavia/octavia-helm-overrides.yaml
@@ -466,7 +466,7 @@ endpoints:
         username: octavia
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /octavia

diff --git a/helm-configs/placement/placement-helm-overrides.yaml b/helm-configs/placement/placement-helm-overrides.yaml
@@ -206,7 +206,7 @@ endpoints:
         username: nova
         password: password
     hosts:
-      default: mariadb-galera-primary
+      default: maxscale-galera
     host_fqdn_override:
       default: null
     path: /placement

diff --git a/kustomize/mariadb-cluster/aio/kustomization.yaml b/kustomize/mariadb-cluster/aio/kustomization.yaml
@@ -12,3 +12,10 @@ patches:
       - op: replace
         path: /spec/galera/enabled
         value: false
+  - target:
+      kind: MaxScale
+      name: maxscale-galera
+    patch: |-
+      - op: replace
+        path: /spec/replicas
+        value: 1
diff --git a/kustomize/mariadb-cluster/base/kustomization.yaml b/kustomize/mariadb-cluster/base/kustomization.yaml
@@ -1,4 +1,5 @@
 resources:
   - mariadb-configmap.yaml
+  - mariadb-maxscale.yaml
   - mariadb-galera.yaml
   - mariadb-backup.yaml
diff --git a/kustomize/mariadb-cluster/base/mariadb-galera.yaml b/kustomize/mariadb-cluster/base/mariadb-galera.yaml
@@ -28,21 +28,8 @@ spec:
     runAsUser: 0
 
   # point to an existing MaxScale instance. Doing this will delegate tasks such as primary failover to MaxScale.
-  # maxScaleRef:
-  #   name: maxscale
-
-  # provision a MaxScale instance and set 'spec.maxScaleRef' automatically.
-  maxScale:
-    enabled: false
-
-    kubernetesService:
-      type: LoadBalancer
-      annotations:
-        metallb.universe.tf/address-pool: primary
-
-    connection:
-      secretName: mxs-galera-conn
-      port: 3306
+  maxScaleRef:
+    name: maxscale-galera
 
   galera:
     enabled: true
@@ -132,8 +119,6 @@ spec:
   resources:
     requests:
       memory: 256Mi
-    limits:
-      memory: 16Gi
 
   metrics:
     enabled: true
diff --git a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml
@@ -0,0 +1,134 @@
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: MaxScale
+metadata:
+  name: maxscale-galera
+spec:
+  replicas: 3
+
+  mariaDbRef:
+    name: mariadb-galera
+    namespace: openstack
+
+  services:
+    - name: rw-router
+      router: readwritesplit
+      params:
+        transaction_replay: "true"
+        transaction_replay_attempts: "10"
+        transaction_replay_timeout: "5s"
+        max_slave_connections: "255"
+        max_replication_lag: "3s"
+        master_accept_reads: "true"
+      listener:
+        name: rw-listener
+        port: 3306
+        protocol: MariaDBProtocol
+        params:
+          connection_metadata: "tx_isolation=auto"
+        suspend: false
+      suspend: false
+    - name: rconn-master-router
+      router: readconnroute
+      params:
+        router_options: "master"
+        max_replication_lag: "3s"
+        master_accept_reads: "true"
+      listener:
+        port: 3307
+    - name: rconn-slave-router
+      router: readconnroute
+      params:
+        router_options: "slave"
+        max_replication_lag: "3s"
+      listener:
+        port: 3308
+
+  monitor:
+    name: mariadb-monitor
+    module: galeramon
+    interval: 2s
+    cooperativeMonitoring: majority_of_all
+    params:
+      disable_master_failback: "false"
+      available_when_donor: "false"
+      disable_master_role_setting: "false"
+    suspend: false
+
+  admin:
+    port: 8989
+    guiEnabled: false
+
+  config:
+    params:
+      log_info: "true"
+    volumeClaimTemplate:
+      resources:
+        requests:
+          storage: 100Mi
+      accessModes:
+        - ReadWriteOnce
+    sync:
+      database: mysql
+      interval: 5s
+      timeout: 10s
+
+  auth:
+    generate: true
+    adminUsername: mariadb-operator
+    adminPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    deleteDefaultAdmin: true
+    clientUsername: maxscale-galera-client
+    clientPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    clientMaxConnections: 90
+    serverUsername: maxscale-galera-server
+    serverPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    serverMaxConnections: 90
+    monitorUsername: maxscale-galera-monitor
+    monitorPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    monitorMaxConnections: 90
+    syncUsername: maxscale-galera-sync
+    syncPasswordSecretKeyRef:
+      name: maxscale
+      key: password
+    syncMaxConnections: 90
+
+  podSecurityContext:
+    runAsUser: 0
+
+  securityContext:
+    allowPrivilegeEscalation: false
+
+  updateStrategy:
+    type: RollingUpdate
+
+  kubernetesService:
+    type: LoadBalancer
+    annotations:
+      metallb.universe.tf/address-pool: primary
+
+  connection:
+    secretName: mxs-galera-conn
+    port: 3306
+
+  resources:
+    requests:
+      memory: 128Mi
+
+  affinity:
+    enableAntiAffinity: true
+
+  tolerations:
+    - key: "k8s.mariadb.com/ha"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  podDisruptionBudget:
+    maxUnavailable: 33%
diff --git a/kustomize/octavia/base/octavia-agent.yaml b/kustomize/octavia/base/octavia-agent.yaml
@@ -81,7 +81,7 @@ spec:
             - name: PATH
               value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/
             - name: DEPENDENCY_SERVICE
-              value: "openstack:mariadb-galera-primary,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server"
+              value: "openstack:maxscale-galera,openstack:keystone-api,openstack:rabbitmq-nodes,openstack:memcached,openstack:neutron-server"
             - name: DEPENDENCY_JOBS
               value: "octavia-db-sync,octavia-ks-user,octavia-ks-endpoints"
             - name: DEPENDENCY_DAEMONSET