fix: Add Afinity where it's needed (#176)

This change ensures our workloads are scheduled to nodes we're expecting. At present, the assumption is that workloads without specific affinity rules would land on "workers" however, that was wrong. This change ensure that workloads such as memcached, mariadb, and rabbitmq are always scheduled to our appropriate workers. Signed-off-by: Kevin Carter <[email protected]>
rackerlabs · Mar 23, 2024 · 5dd7f1a · 5dd7f1a
1 parent 9013dab
commit 5dd7f1a
Show file tree

Hide file tree

Showing 14 changed files with 127 additions and 9 deletions.
diff --git a/kustomize/ingress/external/helm/ingress-helm-overrides.yaml b/kustomize/ingress/external/helm/ingress-helm-overrides.yaml
@@ -65,6 +65,14 @@ pod:
         default: kubernetes.io/hostname
       weight:
         default: 10
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: openstack-control-plane
+                operator: In
+                values:
+                  - enabled
   tolerations:
     ingress:
       enabled: false

diff --git a/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml b/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml
@@ -65,6 +65,14 @@ pod:
         default: kubernetes.io/hostname
       weight:
         default: 10
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: openstack-control-plane
+                operator: In
+                values:
+                  - enabled
   tolerations:
     ingress:
       enabled: false

diff --git a/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml b/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml
@@ -65,6 +65,14 @@ pod:
         default: kubernetes.io/hostname
       weight:
         default: 10
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: openstack-control-plane
+                operator: In
+                values:
+                  - enabled
   tolerations:
     ingress:
       enabled: false

diff --git a/kustomize/mariadb-cluster/base/mariadb-galera.yaml b/kustomize/mariadb-cluster/base/mariadb-galera.yaml
@@ -93,6 +93,14 @@ spec:
 
   affinity:
     enableAntiAffinity: true
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/worker
+                operator: In
+                values:
+                  - worker
 
   tolerations:
     - key: "k8s.mariadb.com/ha"

diff --git a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml
@@ -56,7 +56,7 @@ spec:
 
   admin:
     port: 8989
-    guiEnabled: true
+    guiEnabled: false
 
   config:
     params:
@@ -122,6 +122,14 @@ spec:
 
   affinity:
     enableAntiAffinity: true
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/worker
+                operator: In
+                values:
+                  - worker
 
   tolerations:
     - key: "k8s.mariadb.com/ha"

diff --git a/kustomize/mariadb-operator/kustomization.yaml b/kustomize/mariadb-operator/kustomization.yaml
@@ -11,8 +11,26 @@ helmCharts:
         cert:
           certManager:
             enabled: true
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+                - matchExpressions:
+                    - key: node-role.kubernetes.io/worker
+                      operator: In
+                      values:
+                        - worker
       metrics:
         enabled: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - worker
     includeCRDs: true
     version: 0.27.0
     namespace: mariadb-system
diff --git a/kustomize/memcached/base/kustomization.yaml b/kustomize/memcached/base/kustomization.yaml
@@ -12,5 +12,14 @@ helmCharts:
       persistence:
         enabled: true
         size: 10Gi
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - worker
     includeCRDs: true
     namespace: openstack
diff --git a/kustomize/prometheus-mysql-exporter/values.yaml b/kustomize/prometheus-mysql-exporter/values.yaml
@@ -85,7 +85,15 @@ nodeSelector: {}
 
 tolerations: []
 
-affinity: {}
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/worker
+              operator: In
+              values:
+                - worker
 
 podLabels: {}
 

diff --git a/kustomize/prometheus-postgres-exporter/values.yaml b/kustomize/prometheus-postgres-exporter/values.yaml
@@ -188,7 +188,15 @@ nodeSelector: {}
 
 tolerations: []
 
-affinity: {}
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/worker
+              operator: In
+              values:
+                - worker
 
 annotations: {
   prometheus.io/scrape: "true",

diff --git a/kustomize/prometheus-rabbitmq-exporter/values.yaml b/kustomize/prometheus-rabbitmq-exporter/values.yaml
@@ -32,7 +32,15 @@ nodeSelector: {}
 
 tolerations: []
 
-affinity: {}
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/worker
+              operator: In
+              values:
+                - worker
 
 loglevel: info
 rabbitmq:

diff --git a/kustomize/prometheus/values.yaml b/kustomize/prometheus/values.yaml
@@ -798,7 +798,15 @@ alertmanager:
     ## Assign custom affinity rules to the alertmanager instance
     ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
     ##
-    affinity: {}
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: node-role.kubernetes.io/worker
+                  operator: In
+                  values:
+                    - worker
     # nodeAffinity:
     #   requiredDuringSchedulingIgnoredDuringExecution:
     #     nodeSelectorTerms:

diff --git a/kustomize/rabbitmq-cluster/base/rabbitmq-cluster.yaml b/kustomize/rabbitmq-cluster/base/rabbitmq-cluster.yaml
@@ -6,6 +6,7 @@ metadata:
   annotations:
     metallb.universe.tf/address-pool: pool1
 spec:
+
   replicas: 3
   resources:
     requests:
@@ -33,10 +34,10 @@ spec:
       requiredDuringSchedulingIgnoredDuringExecution:
         nodeSelectorTerms:
           - matchExpressions:
-              - key: openstack-control-plane
+              - key: node-role.kubernetes.io/worker
                 operator: In
                 values:
-                  - enabled
+                  - worker
     # podAntiAffinity:
     #   requiredDuringSchedulingIgnoredDuringExecution:
     #   - labelSelector:

diff --git a/kustomize/sealed-secrets/base/values.yaml b/kustomize/sealed-secrets/base/values.yaml
@@ -204,7 +204,16 @@ runtimeClassName: ""
 ## @param affinity [object] Affinity for Sealed Secret pods assignment
 ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
 ##
-affinity: {}
+affinity:
+  enableAntiAffinity: true
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/worker
+              operator: In
+              values:
+                - worker
 ## @param nodeSelector [object] Node labels for Sealed Secret pods assignment
 ## ref: https://kubernetes.io/docs/user-guide/node-selection/
 ##

diff --git a/kustomize/vault-secrets-operator/base/values.yaml b/kustomize/vault-secrets-operator/base/values.yaml
@@ -54,7 +54,16 @@ controller:
   #           values:
   #           - antarctica-east1
   #           - antarctica-west1
-  affinity: {}
+  affinity:
+    enableAntiAffinity: true
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/worker
+                operator: In
+                values:
+                  - worker
 
   # Settings related to the kubeRbacProxy container. This container is an HTTP proxy for the
   # controller manager which performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.