[obs] Make sure to not alert for 0 backup failures (#20458)

gitpod-io · Dec 16, 2024 · 3e570ae · 3e570ae
1 parent 6b4fae6
commit 3e570ae
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml b/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml
@@ -54,7 +54,9 @@ spec:
         summary: Workspace backups failed recently in cluster {{ $labels.cluster }}
         description: This can happen when a single node has failed in the cloud provider
       expr: |
-        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) <= 16
+        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) > 0
+        AND
+        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) < 16
     - alert: GitpodWsManagerMk2BackupFailureCritical
       labels:
         severity: critical
@@ -64,4 +66,4 @@ spec:
         summary: Workspace backups failed recently in cluster {{ $labels.cluster }}
         description: This can be an indicator of two or more nodes failing in a cloud provider
       expr: |
-        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) > 16
+        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) >= 16