diff --git a/docs/usage.md b/docs/usage.md
index 65c8b51..f70cee7 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -37,6 +37,7 @@ You can find more about scrapeType's on [Scrape Config](https://prometheus.io/do
|alertLabels |This parameter is translated to Prometheus alert `LABELS` statement. It allows specifying a set of additional labels to be attached to the alert. Multiple labels can be separated with comma (`,`).
**Example:** `severity=high,receiver=system`|No|
|alertName |The name of the alert. It is combined with the `serviceName` thus producing an unique identifier.
**Example:** `memoryAlert`|Yes|
|serviceName |The name of the service. It is combined with the `alertName` thus producing an unique identifier.
**Example:** `go-demo`|Yes|
+|alertPersistent|When set to *true*, the alert will persist when the service is scaled to zero replicas.
**Example:** `true`|No|
Those parameters can be indexed so that multiple alerts can be defined for a service. Indexing is sequential and starts from 1. An example of indexed `alertName` could be `alertName.1=memload` and `alertName.2=diskload`.
diff --git a/prometheus/types.go b/prometheus/types.go
index 1b39476..3b94900 100644
--- a/prometheus/types.go
+++ b/prometheus/types.go
@@ -222,6 +222,7 @@ type Alert struct {
AlertIf string `json:"alertIf,omitempty"`
AlertLabels map[string]string `json:"alertLabels,omitempty"`
AlertName string `json:"alertName"`
+ AlertPersistent bool `json:"alertPersistent"`
AlertNameFormatted string
ServiceName string `json:"serviceName"`
Replicas int `json:"replicas"`
diff --git a/server/server.go b/server/server.go
index 1812c3d..760d027 100644
--- a/server/server.go
+++ b/server/server.go
@@ -97,7 +97,7 @@ func (s *serve) ReconfigureHandler(w http.ResponseWriter, req *http.Request) {
logPrintf("Processing " + req.URL.String())
req.ParseForm()
scrape := s.getScrape(req)
- s.deleteAlerts(scrape.ServiceName)
+ s.deleteAlerts(scrape.ServiceName, false)
alerts := s.getAlerts(req)
prometheus.WriteConfig(s.configPath, s.scrapes, s.alerts)
err := prometheus.Reload()
@@ -115,7 +115,7 @@ func (s *serve) RemoveHandler(w http.ResponseWriter, req *http.Request) {
serviceName := req.URL.Query().Get("serviceName")
scrape := s.scrapes[serviceName]
delete(s.scrapes, serviceName)
- alerts := s.deleteAlerts(serviceName)
+ alerts := s.deleteAlerts(serviceName, true)
prometheus.WriteConfig(s.configPath, s.scrapes, s.alerts)
err := prometheus.Reload()
statusCode := http.StatusOK
@@ -241,6 +241,8 @@ func (s *serve) getAlerts(req *http.Request) []prometheus.Alert {
alertName := req.URL.Query().Get(fmt.Sprintf("alertName.%d", i))
annotations := s.getMapFromString(req.URL.Query().Get(fmt.Sprintf("alertAnnotations.%d", i)))
labels := s.getMapFromString(req.URL.Query().Get(fmt.Sprintf("alertLabels.%d", i)))
+ persistant := req.URL.Query().Get(fmt.Sprintf("alertPersistent.%d", i)) == "true"
+
alert := prometheus.Alert{
ServiceName: alertDecode.ServiceName,
AlertName: alertName,
@@ -249,6 +251,7 @@ func (s *serve) getAlerts(req *http.Request) []prometheus.Alert {
AlertAnnotations: annotations,
AlertLabels: labels,
Replicas: replicas,
+ AlertPersistent: persistant,
}
s.formatAlert(&alert)
if !s.isValidAlert(&alert) {
@@ -322,7 +325,6 @@ func GetShortcuts() map[string]AlertIfShortcut {
logPrintf("YAML decoding reading %s, error: %v", path, err)
continue
}
- fmt.Println(secretShortcuts)
for k, v := range secretShortcuts {
shortcuts[k] = v
@@ -483,13 +485,16 @@ func (s *serve) isValidAlert(alert *prometheus.Alert) bool {
return len(alert.AlertName) > 0 && len(alert.AlertIf) > 0
}
-func (s *serve) deleteAlerts(serviceName string) []prometheus.Alert {
+func (s *serve) deleteAlerts(
+ serviceName string, keepPersistantAlerts bool) []prometheus.Alert {
alerts := []prometheus.Alert{}
serviceNameFormatted := s.getNameFormatted(serviceName)
for k, v := range s.alerts {
if strings.HasPrefix(k, serviceNameFormatted) {
- alerts = append(alerts, v)
- delete(s.alerts, k)
+ if !keepPersistantAlerts || !v.AlertPersistent {
+ alerts = append(alerts, v)
+ delete(s.alerts, k)
+ }
}
}
return alerts
diff --git a/server/server_test.go b/server/server_test.go
index 2759fa8..9741fdd 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -231,6 +231,57 @@ func (s *ServerTestSuite) Test_ReconfigureHandler_AddsAlert() {
s.Equal(expected, serve.alerts[expected.AlertNameFormatted])
}
+func (s *ServerTestSuite) Test_ReconfigureHandler_UpdatesPersistantAlert() {
+ // When service is reconfigured, all alert, including persistant alerts, will be removed
+ // and queried again.
+ expected := prometheus.Alert{
+ ServiceName: "my-service",
+ AlertName: "my-alert",
+ AlertIf: "a>b",
+ AlertFor: "my-for",
+ AlertNameFormatted: "myservice_myalert",
+ AlertAnnotations: map[string]string{"a1": "v1", "a2": "v2"},
+ AlertLabels: map[string]string{"l1": "v1"},
+ AlertPersistent: true,
+ }
+ rwMock := ResponseWriterMock{}
+ addr := fmt.Sprintf(
+ "/v1/docker-flow-monitor?serviceName=%s&alertName=%s&alertIf=%s&alertFor=%s&alertAnnotations=%s&alertLabels=%s&alertPersistent=%t",
+ expected.ServiceName,
+ expected.AlertName,
+ url.QueryEscape(expected.AlertIf),
+ expected.AlertFor,
+ url.QueryEscape("a1=v1,a2=v2"),
+ url.QueryEscape("l1=v1"),
+ expected.AlertPersistent,
+ )
+ req, _ := http.NewRequest("GET", addr, nil)
+
+ serve := New()
+ serve.ReconfigureHandler(rwMock, req)
+
+ s.Equal(expected, serve.alerts[expected.AlertNameFormatted])
+
+ // Change alert slightly
+ expected.AlertIf = "a