Added check for inodes usage (#1)

.gitignore

@@ -49,3 +49,5 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 
+alerts/config.ini
+

alerts/bin/check.py

@@ -27,13 +27,16 @@
 
     logging.basicConfig(level=logging.INFO)
 
-    notifier = MailNotifier(
-        name = 'check-status',
-        recipients = aslist(config.get('notify', 'recipients')),
-        mailer = make_mailer(config)
-    )
-
-    #notifier = Notifier('check-status')
+    notifier_name = config.get('notify', 'notifier')
+    notifier = None
+    if notifier_name == 'mailer':
+        notifier = MailNotifier(
+            name = 'check',
+            recipients = aslist(config.get('notify', 'recipients')),
+            mailer = make_mailer(config)
+        )
+    else:
+        notifier = Notifier('check')
 
     hostname = config.get('stats', 'hostname')
     collectd_data_dir = os.path.realpath(config.get('stats', 'collectd_data_dir'))
@@ -84,18 +87,37 @@
     else:
         logging.info('Memory: OK (%.1f < %.1f)' %(u, max_u))
 
-
     # 3. Check filesystem usage
 
     max_u = float(config.get('alerts', 'fs.usage_level'))
     for partition in psutil.disk_partitions():
         mountpoint = partition.mountpoint
+        # Check space
         u = 100.0 * util.get_fs_usage(mountpoint)
         if u > max_u:
             tpl = template_loader.load('fs.excessive-usage.html')
             msg = Message(
                 title = u'Running out of space at %s' %(hostname),
-                summary = u'Used disk space has exceeded %d%%' %(max_u),
+                summary = u'Used disk space has exceeded %d%% at %s' %(max_u, mountpoint),
+                message = tpl.generate(
+                    hostname = hostname,
+                    mountpoint = mountpoint,
+                    max_usage = '%.1f' %(max_u),
+                    avg_usage = '%.1f' %(u),
+                    generated_at = datetime.now()
+                ).render('html')
+            )
+            notifier.add_message(msg, -5)
+            logging.info('Disk(space):%s: FAILED (%.1f > %.1f)' %(mountpoint, u, max_u))
+        else:
+            logging.info('Disk(space):%s: OK (%.1f < %.1f)' %(mountpoint, u, max_u))
+        # Check inodes
+        u = 100.0 * util.get_fs_usage_of_inodes(mountpoint)
+        if u > max_u:
+            tpl = template_loader.load('fs.excessive-usage-of-inodes.html')
+            msg = Message(
+                title = u'Running out of inodes at %s' %(hostname),
+                summary = u'Used inodes have exceeded %d%% at %s' %(max_u, mountpoint),
                 message = tpl.generate(
                     hostname = hostname,
                     mountpoint = mountpoint,
@@ -105,9 +127,9 @@
                 ).render('html')
             )
             notifier.add_message(msg, -5)
-            logging.info('Disk:%s: FAILED (%.1f > %.1f)' %(mountpoint, u, max_u))
+            logging.info('Disk(inodes):%s: FAILED (%.1f > %.1f)' %(mountpoint, u, max_u))
         else:
-            logging.info('Disk:%s: OK (%.1f < %.1f)' %(mountpoint, u, max_u))
+            logging.info('Disk(inodes):%s: OK (%.1f < %.1f)' %(mountpoint, u, max_u))
 
     # 4. Check Nginx workload (active connections)
 

alerts/config.ini → alerts/config-example.ini

@@ -20,13 +20,15 @@ nginx.active_connections = 100
 
 [mailer]
 
-smtp_host = mail.dsa.gr
+smtp_host = mail.example.com
 smtp_port = 465
 
-smtp_user = admin@olomeleia.gr
-smtp_pass = 131313
+smtp_user = admin@example.com
+smtp_pass = secret
 
 [notify]
 
+notifier = mailer
+
 recipients = [email protected]
 

alerts/lib/notify.py

@@ -36,7 +36,7 @@ def notify(self):
 
 class MailNotifier(Notifier):
 
-    MAX_NUM_EMAILS = 1
+    MAX_NUM_EMAILS = 2
 
     def __init__(self, name, recipients, mailer):
         assert isinstance(mailer,Mailer), 'Expected a 2nd argument of type %s' %(Mailer)

alerts/lib/util.py

@@ -35,15 +35,25 @@ def get_memory_free_kilobytes(start='-600s', resolution='120'):
     return (free_bytes / 1024);
 
 def get_fs_usage(mountpoint, start='-1200s', resolution='600'):
-    info = psutil.disk_usage(mountpoint)
+    info = os.statvfs(mountpoint)
     name = mountpoint.strip('/')
     name = 'root' if (not len(name)) else name.replace('/','-')
     rrd_file = os.path.join(COLLECTD_DATA_DIR, 'df-%s/df_complex-free.rrd' %(name))
     disk_stats = collected_stats.DfStats(rrd_file)
     free_bytes = disk_stats.avg('value', start, resolution)
-    total_bytes = info.total
+    total_bytes = info.f_blocks * info.f_bsize
     return (1 - (free_bytes/total_bytes))
 
+def get_fs_usage_of_inodes(mountpoint, start='-1200s', resolution='600'):
+    info = os.statvfs(mountpoint)
+    name = mountpoint.strip('/')
+    name = 'root' if (not len(name)) else name.replace('/','-')
+    rrd_file = os.path.join(COLLECTD_DATA_DIR, 'df-%s/df_inodes-free.rrd' %(name))
+    disk_stats = collected_stats.DfStats(rrd_file)
+    free_inodes = disk_stats.avg('value', start, resolution)
+    max_inodes = info.f_files
+    return (1 - (free_inodes/max_inodes))
+
 def get_fs_free_kilobytes(mountpoint, start='-1200s', resolution='600'):
     name = mountpoint.strip('/')
     name = 'root' if (not len(name)) else name.replace('/','-')

alerts/templates/fs.excessive-usage-of-inodes.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://genshi.edgewall.org/">
+  <head>
+  </head>
+  <body>
+    <div id="header">
+      <h4>Running out of inodes at <em>${hostname}</em></h4>
+    </div>
+
+    <div id="message">
+        <p>The filesystem resources on <code>${mountpoint}</code> are exhausting: exceeded the usage limit of ${max_usage}%. The average usage was ${avg_usage}%. </p>
+    </div>
+
+    <div py:if="defined('generated_at')" id="footer">
+        <p><small>Report generated at ${generated_at.isoformat()}</small></p>
+    </div>
+  </body>
+</html>