Add alertmanager to report alerts based on metricly metrics

README.md

@@ -23,6 +23,8 @@
   - Logs incoming and outgoing API requests with support for multiple log levels (INFO, DEBUG, ERROR).
 - **Metrics Visualization**
   - Provides an inbuilt `Grafana` dashboard to visualize all metrics.
+- **Alerting Mechanism**
+  - Provides an inbuilt `Alertmanager` rules that send Gmail alerts.
 ---
 
 ## **Getting Started**
@@ -371,6 +373,17 @@ The Metricly exporter provides the following API endpoints:
       }
       ```      
 
+### **Alertmanager Configuration** ###
+Metricly provides a few inbuilt alerts to monitor high utilization of CPU, Memory and Disk usage.
+
+![Sample Alerts](doc/alerts.png)
+
+Upon meeting condition for any alert, an email notification is sent to the receiver configured in `config/alertmanager/alertmanager.yml`
+
+![High CPU Alert](doc/high_cpu_alert.png)
+
+To include more alerts, take a look at `config/prometheus/alerts/`. Similar alerts can be built and added to the same directory.
+
 ### **Development**
 
 #### **Testing**

alertmanager.yml

@@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'default'
+
+receivers:
+  - name: 'default'
+    email_configs:
+      - to: '[email protected]'
+        from: '[email protected]'
+        smarthost: 'smtp.gmail.com:587'
+        auth_username: '[email protected]'
+        auth_identity: '[email protected]'
+        auth_password: 'jqhotwxphmieuaqw'

config/alertmanager/alertmanager.yml

@@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'default'
+
+receivers:
+  - name: 'default'
+    email_configs:
+      - to: '[email protected]'           # receiver's address
+        from: '[email protected]'   
+        smarthost: 'smtp.gmail.com:587'
+        auth_username: '[email protected]' # sender's address
+        auth_identity: '[email protected]' # sender's address
+        auth_password: 'xxxxxxxxxxxxx'    # gmail app password generated by sender

config/prometheus/alerts/cpu_alerts.yml

@@ -0,0 +1,20 @@
+groups:
+  - name: cpu_alerts
+    rules:
+      - alert: CPUUsage > 60%
+        expr: avg_over_time(metricly_cpu_total[5m]) > 60
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU usage detected"
+          description: "CPU usage is above 60% for the last 5 minutes on host {{ $labels.host }}"
+
+      - alert: CPUUsage > 80%
+        expr: avg_over_time(metricly_cpu_total[15m]) > 80
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "High CPU usage detected"
+          description: "CPU usage is above 80% for the last 15 minutes on host {{ $labels.host }}"

config/prometheus/alerts/disk_alerts.yml

@@ -0,0 +1,20 @@
+groups:
+  - name: disk_alerts
+    rules:
+      - alert: Disk Usage > 60%
+        expr: 100*metricly_disk_used_bytes/metricly_disk_total_bytes > 60
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High Disk usage detected"
+          description: "Disk usage is above 60%"
+
+      - alert: Disk Usage > 80%
+        expr: 100*metricly_disk_used_bytes/metricly_disk_total_bytes > 80
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "High Disk usage detected"
+          description: "Disk usage is above 80%"

config/prometheus/alerts/memory_alerts.yml

@@ -0,0 +1,20 @@
+groups:
+  - name: memory_alerts
+    rules:
+      - alert: Memory Usage > 60%
+        expr: 100*(metricly_memory_total_bytes-metricly_memory_available_bytes)/metricly_memory_total_bytes > 60
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High Memory usage detected"
+          description: "Memory usage is above 60%"
+
+      - alert: Memory Usage > 80%
+        expr: 100*(metricly_memory_total_bytes-metricly_memory_available_bytes)/metricly_memory_total_bytes > 80
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "High Memory usage detected"
+          description: "Memory usage is above 80%"

docker-compose.yml

@@ -54,3 +54,12 @@ services:
     depends_on:
       - prometheus
 
+  alertmanager:
+    container_name: metricly_alertmanager
+    image: docker.io/prom/alertmanager:v0.27.0
+    network_mode: host
+    volumes:
+      - ./config/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro,z
+    restart: always
+    depends_on:
+      - prometheus