Add cluster_control_plane_unhealthy inhibition. (#1607)

* Add `cluster_control_plane_unhealthy` inhibition. * updated test cases
giantswarm · May 6, 2024 · d4b6911 · d4b6911
1 parent 2109d67
commit d4b6911
Show file tree

Hide file tree

Showing 26 changed files with 155 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,12 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add `cluster_control_plane_unhealthy` inhibition.
+
 ## [4.74.0] - 2024-05-02
 
 ### Changed
 
 - Expose prometheus agent sharding strategies as prometheus-meta-operator configuration parameters so we can experiment with the scaling strategies.
-- 
+
 ## [4.73.1] - 2024-05-01
 
 ### Fixed

diff --git a/files/templates/alertmanager/alertmanager.yaml b/files/templates/alertmanager/alertmanager.yaml
@@ -611,6 +611,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...e/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-1-capa-mc.golden b/...e/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-1-capa-mc.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...urce/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-2-capa.golden b/...urce/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-2-capa.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...urce/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-3-capz.golden b/...urce/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-3-capz.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...ource/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-4-eks.golden b/...ource/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-4-eks.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...ource/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-5-gcp.golden b/...ource/alerting/alertmanagerconfig/test/alertmanager-config/classic/capi/case-5-gcp.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...ting/alertmanagerconfig/test/alertmanager-config/classic/vintage/case-1-vintage-mc.golden b/...ting/alertmanagerconfig/test/alertmanager-config/classic/vintage/case-1-vintage-mc.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...lerting/alertmanagerconfig/test/alertmanager-config/classic/vintage/case-2-aws-v16.golden b/...lerting/alertmanagerconfig/test/alertmanager-config/classic/vintage/case-2-aws-v16.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...lerting/alertmanagerconfig/test/alertmanager-config/classic/vintage/case-3-aws-v18.golden b/...lerting/alertmanagerconfig/test/alertmanager-config/classic/vintage/case-3-aws-v18.golden
@@ -420,6 +420,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...ting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-1-capa-mc.golden b/...ting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-1-capa-mc.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...lerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-2-capa.golden b/...lerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-2-capa.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...lerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-3-capz.golden b/...lerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-3-capz.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...alerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-4-eks.golden b/...alerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-4-eks.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...alerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-5-gcp.golden b/...alerting/alertmanagerconfig/test/alertmanager-config/mimir-enabled/capi/case-5-gcp.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...lertmanagerconfig/test/alertmanager-config/mimir-enabled/vintage/case-1-vintage-mc.golden b/...lertmanagerconfig/test/alertmanager-config/mimir-enabled/vintage/case-1-vintage-mc.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...g/alertmanagerconfig/test/alertmanager-config/mimir-enabled/vintage/case-2-aws-v16.golden b/...g/alertmanagerconfig/test/alertmanager-config/mimir-enabled/vintage/case-2-aws-v16.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...g/alertmanagerconfig/test/alertmanager-config/mimir-enabled/vintage/case-3-aws-v18.golden b/...g/alertmanagerconfig/test/alertmanager-config/mimir-enabled/vintage/case-3-aws-v18.golden
@@ -439,6 +439,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...erting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-1-capa-mc.golden b/...erting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-1-capa-mc.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/.../alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-2-capa.golden b/.../alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-2-capa.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/.../alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-3-capz.golden b/.../alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-3-capz.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...e/alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-4-eks.golden b/...e/alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-4-eks.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...e/alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-5-gcp.golden b/...e/alerting/alertmanagerconfig/test/alertmanager-config/slack-token/capi/case-5-gcp.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/.../alertmanagerconfig/test/alertmanager-config/slack-token/vintage/case-1-vintage-mc.golden b/.../alertmanagerconfig/test/alertmanager-config/slack-token/vintage/case-1-vintage-mc.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...ing/alertmanagerconfig/test/alertmanager-config/slack-token/vintage/case-2-aws-v16.golden b/...ing/alertmanagerconfig/test/alertmanager-config/slack-token/vintage/case-2-aws-v16.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers:

diff --git a/...ing/alertmanagerconfig/test/alertmanager-config/slack-token/vintage/case-3-aws-v18.golden b/...ing/alertmanagerconfig/test/alertmanager-config/slack-token/vintage/case-3-aws-v18.golden
@@ -460,6 +460,12 @@ inhibit_rules:
   - cancel_if_cluster_with_notready_nodepools=true
   equal: [cluster_id]
 
+- source_matchers:
+  - cluster_control_plane_unhealthy=true
+  target_matchers:
+  - cancel_if_cluster_control_plane_unhealthy=true
+  equal: [cluster_id]
+
 - source_matchers:
   - instance_state_not_running=true
   target_matchers: