From 5dd7f1a58ad7131b33d4bbd93308d2df49ab4a72 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Sat, 23 Mar 2024 13:58:00 -0500 Subject: [PATCH] fix: Add Afinity where it's needed (#176) This change ensures our workloads are scheduled to nodes we're expecting. At present, the assumption is that workloads without specific affinity rules would land on "workers" however, that was wrong. This change ensure that workloads such as memcached, mariadb, and rabbitmq are always scheduled to our appropriate workers. Signed-off-by: Kevin Carter --- .../external/helm/ingress-helm-overrides.yaml | 8 ++++++++ .../grafana/helm/ingress-helm-overrides.yaml | 8 ++++++++ .../internal/helm/ingress-helm-overrides.yaml | 8 ++++++++ .../mariadb-cluster/base/mariadb-galera.yaml | 8 ++++++++ .../mariadb-cluster/base/mariadb-maxscale.yaml | 10 +++++++++- kustomize/mariadb-operator/kustomization.yaml | 18 ++++++++++++++++++ kustomize/memcached/base/kustomization.yaml | 9 +++++++++ .../prometheus-mysql-exporter/values.yaml | 10 +++++++++- .../prometheus-postgres-exporter/values.yaml | 10 +++++++++- .../prometheus-rabbitmq-exporter/values.yaml | 10 +++++++++- kustomize/prometheus/values.yaml | 10 +++++++++- .../base/rabbitmq-cluster.yaml | 5 +++-- kustomize/sealed-secrets/base/values.yaml | 11 ++++++++++- .../vault-secrets-operator/base/values.yaml | 11 ++++++++++- 14 files changed, 127 insertions(+), 9 deletions(-) diff --git a/kustomize/ingress/external/helm/ingress-helm-overrides.yaml b/kustomize/ingress/external/helm/ingress-helm-overrides.yaml index d1f13ff4..f8c3dec5 100644 --- a/kustomize/ingress/external/helm/ingress-helm-overrides.yaml +++ b/kustomize/ingress/external/helm/ingress-helm-overrides.yaml @@ -65,6 +65,14 @@ pod: default: kubernetes.io/hostname weight: default: 10 + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: openstack-control-plane + operator: In + values: + - enabled tolerations: ingress: enabled: false diff --git a/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml b/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml index d30c3755..7eb60a5b 100644 --- a/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml +++ b/kustomize/ingress/grafana/helm/ingress-helm-overrides.yaml @@ -65,6 +65,14 @@ pod: default: kubernetes.io/hostname weight: default: 10 + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: openstack-control-plane + operator: In + values: + - enabled tolerations: ingress: enabled: false diff --git a/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml b/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml index c196fa5f..caf50d85 100644 --- a/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml +++ b/kustomize/ingress/internal/helm/ingress-helm-overrides.yaml @@ -65,6 +65,14 @@ pod: default: kubernetes.io/hostname weight: default: 10 + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: openstack-control-plane + operator: In + values: + - enabled tolerations: ingress: enabled: false diff --git a/kustomize/mariadb-cluster/base/mariadb-galera.yaml b/kustomize/mariadb-cluster/base/mariadb-galera.yaml index aa737aa3..d77fdeaf 100644 --- a/kustomize/mariadb-cluster/base/mariadb-galera.yaml +++ b/kustomize/mariadb-cluster/base/mariadb-galera.yaml @@ -93,6 +93,14 @@ spec: affinity: enableAntiAffinity: true + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker tolerations: - key: "k8s.mariadb.com/ha" diff --git a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml index 350d8ca0..362d2f2f 100644 --- a/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml +++ b/kustomize/mariadb-cluster/base/mariadb-maxscale.yaml @@ -56,7 +56,7 @@ spec: admin: port: 8989 - guiEnabled: true + guiEnabled: false config: params: @@ -122,6 +122,14 @@ spec: affinity: enableAntiAffinity: true + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker tolerations: - key: "k8s.mariadb.com/ha" diff --git a/kustomize/mariadb-operator/kustomization.yaml b/kustomize/mariadb-operator/kustomization.yaml index 4f78600f..dc9c9148 100644 --- a/kustomize/mariadb-operator/kustomization.yaml +++ b/kustomize/mariadb-operator/kustomization.yaml @@ -11,8 +11,26 @@ helmCharts: cert: certManager: enabled: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker metrics: enabled: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker includeCRDs: true version: 0.27.0 namespace: mariadb-system diff --git a/kustomize/memcached/base/kustomization.yaml b/kustomize/memcached/base/kustomization.yaml index 3746fd44..105aaee9 100644 --- a/kustomize/memcached/base/kustomization.yaml +++ b/kustomize/memcached/base/kustomization.yaml @@ -12,5 +12,14 @@ helmCharts: persistence: enabled: true size: 10Gi + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker includeCRDs: true namespace: openstack diff --git a/kustomize/prometheus-mysql-exporter/values.yaml b/kustomize/prometheus-mysql-exporter/values.yaml index affb28f0..65be7878 100644 --- a/kustomize/prometheus-mysql-exporter/values.yaml +++ b/kustomize/prometheus-mysql-exporter/values.yaml @@ -85,7 +85,15 @@ nodeSelector: {} tolerations: [] -affinity: {} +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker podLabels: {} diff --git a/kustomize/prometheus-postgres-exporter/values.yaml b/kustomize/prometheus-postgres-exporter/values.yaml index 01ebe1b9..ccb0490d 100644 --- a/kustomize/prometheus-postgres-exporter/values.yaml +++ b/kustomize/prometheus-postgres-exporter/values.yaml @@ -188,7 +188,15 @@ nodeSelector: {} tolerations: [] -affinity: {} +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker annotations: { prometheus.io/scrape: "true", diff --git a/kustomize/prometheus-rabbitmq-exporter/values.yaml b/kustomize/prometheus-rabbitmq-exporter/values.yaml index 3a63cb83..427fb467 100644 --- a/kustomize/prometheus-rabbitmq-exporter/values.yaml +++ b/kustomize/prometheus-rabbitmq-exporter/values.yaml @@ -32,7 +32,15 @@ nodeSelector: {} tolerations: [] -affinity: {} +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker loglevel: info rabbitmq: diff --git a/kustomize/prometheus/values.yaml b/kustomize/prometheus/values.yaml index 8f579718..83f118ae 100644 --- a/kustomize/prometheus/values.yaml +++ b/kustomize/prometheus/values.yaml @@ -798,7 +798,15 @@ alertmanager: ## Assign custom affinity rules to the alertmanager instance ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ## - affinity: {} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: diff --git a/kustomize/rabbitmq-cluster/base/rabbitmq-cluster.yaml b/kustomize/rabbitmq-cluster/base/rabbitmq-cluster.yaml index 8ee80fb6..5b05bd07 100644 --- a/kustomize/rabbitmq-cluster/base/rabbitmq-cluster.yaml +++ b/kustomize/rabbitmq-cluster/base/rabbitmq-cluster.yaml @@ -6,6 +6,7 @@ metadata: annotations: metallb.universe.tf/address-pool: pool1 spec: + replicas: 3 resources: requests: @@ -33,10 +34,10 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: openstack-control-plane + - key: node-role.kubernetes.io/worker operator: In values: - - enabled + - worker # podAntiAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # - labelSelector: diff --git a/kustomize/sealed-secrets/base/values.yaml b/kustomize/sealed-secrets/base/values.yaml index a4172609..15524ee2 100644 --- a/kustomize/sealed-secrets/base/values.yaml +++ b/kustomize/sealed-secrets/base/values.yaml @@ -204,7 +204,16 @@ runtimeClassName: "" ## @param affinity [object] Affinity for Sealed Secret pods assignment ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## -affinity: {} +affinity: + enableAntiAffinity: true + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker ## @param nodeSelector [object] Node labels for Sealed Secret pods assignment ## ref: https://kubernetes.io/docs/user-guide/node-selection/ ## diff --git a/kustomize/vault-secrets-operator/base/values.yaml b/kustomize/vault-secrets-operator/base/values.yaml index dee524fe..99feb554 100644 --- a/kustomize/vault-secrets-operator/base/values.yaml +++ b/kustomize/vault-secrets-operator/base/values.yaml @@ -54,7 +54,16 @@ controller: # values: # - antarctica-east1 # - antarctica-west1 - affinity: {} + affinity: + enableAntiAffinity: true + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - worker # Settings related to the kubeRbacProxy container. This container is an HTTP proxy for the # controller manager which performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.