Add on_failure to health check

README.md

@@ -210,8 +210,10 @@ services:
             headers:
               Host: domain.org
               Authorization: "Bearer abc123"
-            # on_failure options will be added in the future
-
+
+            # on_failure runs a shell command if the check fails. Expands $date, $error, $check_name.
+            on_failure: |
+              curl -d "Health check '$check_name' failed at $date due to: $error" ntfy.sh/gatego
         cache: true  # Cache responses that has cache headers (Cache-Control and Expire)
 
 ```

checker.go

@@ -4,19 +4,22 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/hvuhsg/gatego/pkg/cron"
 )
 
 type Check struct {
-	Name    string
-	Cron    string
-	URL     string
-	Method  string
-	Timeout time.Duration
-	Headers map[string]string
+	Name      string
+	Cron      string
+	URL       string
+	Method    string
+	Timeout   time.Duration
+	Headers   map[string]string
+	OnFailure string
 }
 
 func (c Check) run(onFailure func(error)) func() {
@@ -57,18 +60,40 @@ func (c Check) run(onFailure func(error)) func() {
 	}
 }
 
+func handleFailure(check Check, err error) error {
+	// Expand command
+	command := check.OnFailure
+	date := time.Now().UTC().Format("2006-01-02 15:04:05")
+	command = strings.ReplaceAll(command, "$date", date)
+	command = strings.ReplaceAll(command, "$error", err.Error())
+	command = strings.ReplaceAll(command, "$check_name", check.Name)
+
+	// Run it
+	args := strings.Split(command, " ")
+	cmd := exec.Command(args[0], args[1:]...)
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	return nil
+}
+
 type Checker struct {
 	Delay     time.Duration
 	Checks    []Check
 	scheduler *cron.Cron
-	OnFailure func(error)
 }
 
 func (c Checker) Start() error {
 	c.scheduler = cron.New()
 
 	for _, check := range c.Checks {
-		err := c.scheduler.Add(uuid.NewString(), check.Cron, check.run(c.OnFailure))
+		err := c.scheduler.Add(uuid.NewString(), check.Cron, check.run(func(err error) {
+			if check.OnFailure != "" {
+				if err := handleFailure(check, err); err != nil {
+					log.Default().Printf("Failed to spawn on_failure command: %s\n", err)
+				}
+			}
+		}))
 		if err != nil {
 			return err
 		}

checker_test.go

@@ -1,6 +1,7 @@
 package gatego
 
 import (
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -140,6 +141,67 @@ func TestChecker_Start(t *testing.T) {
 	}
 }
 
+func TestChecker_OnFailure(t *testing.T) {
+	tests := []struct {
+		name          string
+		checker       Checker
+		expectedError bool
+	}{
+		{
+			name: "on failure command with valid command",
+			checker: Checker{
+				Delay: 1 * time.Second,
+				Checks: []Check{
+					{
+						Name:      "test-check-failure",
+						Cron:      "* * * * *",
+						Method:    "GET",
+						URL:       "http://example.com",
+						Timeout:   5 * time.Second,
+						OnFailure: "echo check '$check_name' failed at $date: $error",
+					},
+				},
+			},
+			expectedError: false,
+		},
+		{
+			name: "on failure command with invalid command",
+			checker: Checker{
+				Delay: 1 * time.Second,
+				Checks: []Check{
+					{
+						Name:      "test-check-failure",
+						Cron:      "* * * * *",
+						Method:    "GET",
+						URL:       "http://example.com",
+						Timeout:   5 * time.Second,
+						OnFailure: "invalidCommand $error",
+					},
+				},
+			},
+			expectedError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Simulate a failure scenario by injecting an error
+			err := errors.New("Connection timeout")
+			err = handleFailure(tt.checker.Checks[0], err)
+
+			// Check if an error was returned and if it matches the expected result
+			if (err != nil) != tt.expectedError {
+				t.Errorf("handleFailure() error = %v, expectedError %v", err, tt.expectedError)
+			}
+
+			// Clean up scheduler if it was created
+			if tt.checker.scheduler != nil {
+				tt.checker.scheduler.Stop()
+			}
+		})
+	}
+}
+
 // TestCheckWithMockServer tests the Check struct with a mock HTTP server
 func TestCheckWithMockServer(t *testing.T) {
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

cmd/config.yaml

@@ -44,6 +44,8 @@ services:
             headers:
               Host: domain.org
               Authorization: "Bearer abc123"
+            on_failure: |
+              echo Health check '$check_name' failed at $date with error: $error
 
         omit_headers: [Authorization, X-API-Key, X-Secret-Token]
 

config-schema.json

@@ -227,6 +227,13 @@
 														"Authorization": "Bearer abc123"
 													}
 												]
+											},
+											"on_failure": {
+												"type": "string",
+												"description": "Shell command to execute if the health check fails. Supports variable expansion: $date, $error, and $check_name.",
+												"examples": [
+													"echo Health check '$check_name' failed at $date with error: $error"
+												]
 											}
 										}
 									}

config/config.go

@@ -51,12 +51,13 @@ func (b Backend) validate() error {
 }
 
 type Check struct {
-	Name    string            `yaml:"name"`
-	Cron    string            `yaml:"cron"`
-	URL     string            `yaml:"url"`
-	Method  string            `yaml:"method"`
-	Timeout time.Duration     `yaml:"timeout"`
-	Headers map[string]string `yaml:"headers"`
+	Name      string            `yaml:"name"`
+	Cron      string            `yaml:"cron"`
+	URL       string            `yaml:"url"`
+	Method    string            `yaml:"method"`
+	Timeout   time.Duration     `yaml:"timeout"`
+	Headers   map[string]string `yaml:"headers"`
+	OnFailure string            `yaml:"on_failure"`
 }
 
 func (c Check) validate() error {

gatego.go

@@ -71,18 +71,19 @@ func (gg GateGo) Run() error {
 }
 
 func createChecker(services []config.Service) *Checker {
-	checker := &Checker{Delay: 5 * time.Second, OnFailure: func(err error) {}}
+	checker := &Checker{Delay: 5 * time.Second}
 
 	for _, service := range services {
 		for _, path := range service.Paths {
 			for _, checkConfig := range path.Checks {
 				check := Check{
-					Name:    checkConfig.Name,
-					Cron:    checkConfig.Cron,
-					URL:     checkConfig.URL,
-					Method:  checkConfig.Method,
-					Timeout: checkConfig.Timeout,
-					Headers: checkConfig.Headers,
+					Name:      checkConfig.Name,
+					Cron:      checkConfig.Cron,
+					URL:       checkConfig.URL,
+					Method:    checkConfig.Method,
+					Timeout:   checkConfig.Timeout,
+					Headers:   checkConfig.Headers,
+					OnFailure: checkConfig.OnFailure,
 				}
 
 				checker.Checks = append(checker.Checks, check)