Try machine low alert again.

metal-stack · Sep 21, 2023 · 2749d61 · 2749d61
1 parent 4dbbf3d
commit 2749d61
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml b/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml
@@ -292,12 +292,12 @@ additionalPrometheusRulesMap:
         annotations:
           description: "Partition {{ $labels.partition }} has {{ $value }} DEAD Machines"
       - alert: MachineCapacityLow
-        expr: avg(metal_partition_capacity_free{size!="unknown"} > 5) by (partition, size) / avg(metal_partition_capacity_total{size!="unknown"} > 5) by (partition, size) < 0.10
+        expr: (avg(metal_partition_capacity_free{size!="unknown"} > 5) by (partition, size) / avg(metal_partition_capacity_total{size!="unknown"} > 5) by (partition, size)) * 100 < 80
         for: 10m
         labels:
           severity: "warning"
         annotations:
-          description: "{{ $value }}% of the {{ $labels.size }} machines in {{ $labels.partition }} are in use."
+          description: "Running out of machines. Less than 80% free machines of size {{ $labels.size }} in {{ $labels.partition }}. (only {{ $value }}% are free)"
       - alert: InternetIPCapacityLow
         expr: avg(metal_network_ip_used{networkId=~"internet.*"}) by (partition, networkId) / avg(metal_network_ip_available{networkId=~"internet.*"}) by (partition, networkId) * 100 > 80
         for: 10m