Add separate alert for HTTP gateway timeout responses

nano-byte · Jun 21, 2024 · 1c2d3b7 · 1c2d3b7
1 parent 28b8ab3
commit 1c2d3b7
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 3 deletions.
diff --git a/charts/generic-service/README.md b/charts/generic-service/README.md
@@ -157,7 +157,8 @@ app:
 | `alerting.http.referenceInterval`               | `1w`                        | The time interval to to compare with the sample interval to detect changes                               |
 | `alerting.http.maxSlowdown`                     | `2.5`                       | The maximum HTTP response slowdown in the sample interval compared to the reference interval             |
 | `alerting.http.max4xxRatio`                     | `2.5`                       | The maximum HTTP 4xx ratio increase in the sample interval compared to the reference interval            |
-| `alerting.http.max5xxCount`                     | `0`                         | The maximum number of HTTP 5xx responses in the sample interval                                          |
+| `alerting.http.max5xxCount`                     | `0`                         | The maximum number of HTTP 5xx responses (except 504) in the sample interval                             |
+| `alerting.http.maxTimeoutCount`                 | `0`                         | The maximum number of HTTP gateway timeout responses (504) in the sample interval                        |
 | `alerting.grpc.requestsMetric`                  | `grpc_server_handled_total` | The name of the Prometheus metric counting gRPC requests                                                 |
 | `alerting.grpc.sampleInterval`                  | `20m`                       | The time interval in which to measure gRPC responses                                                     |
 | `alerting.grpc.referenceInterval`               | `1w`                        | The time interval to to compare with the sample interval to detect changes                               |

diff --git a/charts/generic-service/templates/alerts.yaml b/charts/generic-service/templates/alerts.yaml
@@ -178,12 +178,21 @@ spec:
 
         - alert: Http5xx
           expr: |
-            sum(round(increase({{ include "generic-service.request-code-count-metric" . }}"5.."}[{{ .Values.alerting.http.sampleInterval }}])))
+            sum(round(increase({{ include "generic-service.request-code-count-metric" . }}"5.[^4]"}[{{ .Values.alerting.http.sampleInterval }}])))
             > {{ .Values.alerting.http.max5xxCount }}
           labels: {{- include "generic-service.alert-labels" . | nindent 12 }} critical
             topic: ingress
           annotations: {{- include "generic-service.alert-annotations" . | nindent 12 }} HTTP 5xx responses
             description: '{{ include "generic-service.fullname" . }} gave {{"{{ $value }}"}} HTTP 5xx responses in the last {{ .Values.alerting.http.sampleInterval }}.'
+
+        - alert: HttpTimeout
+          expr: |
+            sum(round(increase({{ include "generic-service.request-code-count-metric" . }}"504"}[{{ .Values.alerting.http.sampleInterval }}])))
+            > {{ .Values.alerting.http.maxTimoutCount }}
+          labels: {{- include "generic-service.alert-labels" . | nindent 12 }} critical
+            topic: ingress
+          annotations: {{- include "generic-service.alert-annotations" . | nindent 12 }} HTTP gateway timeout responses
+            description: '{{ include "generic-service.fullname" . }} gave {{"{{ $value }}"}} HTTP gateway timout responses in the last {{ .Values.alerting.http.sampleInterval }}.'
         {{- end }}
 
         {{- if or (eq .Values.ingress.protocol "grpc") (eq .Values.ingress.protocol "grpcs") }}

diff --git a/charts/generic-service/values.schema.json b/charts/generic-service/values.schema.json
@@ -889,7 +889,12 @@
             "max5xxCount": {
               "type": "number",
               "default": 0,
-              "description": "The maximum number of HTTP 5xx responses in the sample interval"
+              "description": "The maximum number of HTTP 5xx responses (except 504) in the sample interval"
+            },
+            "maxTimeoutCount": {
+              "type": "number",
+              "default": 0,
+              "description": "The maximum number of HTTP gateway timeout responses (504) in the sample interval"
             }
           },
           "additionalProperties": false

diff --git a/charts/generic-service/values.yaml b/charts/generic-service/values.yaml
@@ -173,6 +173,7 @@ alerting:
     maxSlowdown: 2.5
     max4xxRatio: 2.5
     max5xxCount: 0
+    maxTimeoutCount: 0
   grpc:
     requestsMetric: grpc_server_handled_total
     sampleInterval: 20m