From 21eec8e3599e61ff67115b744ce59fc973913b72 Mon Sep 17 00:00:00 2001
From: pgraziano <pgraziano@us.ibm.com>
Date: Wed, 16 May 2018 23:14:00 +0000
Subject: [PATCH] update logstash check to for old indexes that were not
 curated

---
 sensu/plugins/check-logstash-index-dates.sh | 57 ++++++++++---
 sensu/plugins/check-mtime.rb                | 92 ---------------------
 2 files changed, 47 insertions(+), 102 deletions(-)
 mode change 100644 => 100755 sensu/plugins/check-logstash-index-dates.sh
 delete mode 100644 sensu/plugins/check-mtime.rb

diff --git a/sensu/plugins/check-logstash-index-dates.sh b/sensu/plugins/check-logstash-index-dates.sh
old mode 100644
new mode 100755
index 378a144..cdf6b16
--- a/sensu/plugins/check-logstash-index-dates.sh
+++ b/sensu/plugins/check-logstash-index-dates.sh
@@ -2,20 +2,57 @@
 #
 # This runs on elk hosts.
 # Checks to ensure there are no logstash indexes with a future date.
+# Checks to ensure curator has deleted old indexes.
 #
 
-TODAY=$(date -I)
+declare -a ERR_MSGS
 
-let "count=0"
-for index in $(curl -s localhost:9200/_cat/indices?v  | grep -P '\d\d\d\d.\d\d.\d\d' | awk '{print $3}' | sed 's/.*-//;s/\./-/g;s/ .*//' | sort -nk1); do
-    if [[ $index > $TODAY ]]; then
-        let "count++"
+# joins elements of an array with the given character
+function join_by { local IFS="$1"; shift; echo "$*"; }
+
+# checks for indexes with future dates
+function find_future_indexes {
+    TODAY=$(date -I)
+    count=0
+    for index in $(curl -s localhost:9200/_cat/indices?v  | grep -P '\d\d\d\d.\d\d.\d\d' | awk '{print $3}' | sed 's/.*-//;s/\./-/g;s/ .*//' | sort -nk1); do
+        if [[ $index > $TODAY ]]; then
+            count=$(( $count + 1 ))
+        fi
+    done
+
+    if [[ $count -gt 0 ]]; then
+        ERR_MSGS+=("$count elasticsearch index(es) with future dates were found.")
     fi
-done
+}
+
+# checks for old indexes that were not curated
+function find_old_indexes {
+    for file in /etc/elasticsearch/delete_*.yml; do
+        INDEX_PREFIX=$(awk '/value/ {print $2}' $file | sed 's/-$//')
+        CUTOFF_DAYS_AGO=$(awk '/unit_count/ {print $2}' $file)
+        CUTOFF_DATE=`date --date="$(( $CUTOFF_DAYS_AGO + 1 )) day ago" +%Y.%m.%d`
+
+        count=0
+        for i in $(curl -s 'localhost:9200/_cat/indices' | grep -E "$INDEX_PREFIX-[0-9]{4}.[0-9]{2}.[0-9]{2}" | awk '{print $3}' | sort -n); do
+            if [[ ${i##$INDEX_PREFIX-} < $CUTOFF_DATE ]]; then
+                count=$(( $count + 1 ))
+                #echo "$i is older than $CUTOFF_DATE"
+            fi
+        done
 
-if [[ $count -gt 0 ]]; then
-    echo "CRITICAL: $count elasticsearch index(es) with future dates were found."
+        if [[ $count -gt 0 ]]; then
+            ERR_MSGS+=("$count $INDEX_PREFIX index(es) older than $CUTOFF_DAYS_AGO days found.")
+        fi
+    done
+}
+
+find_future_indexes
+find_old_indexes
+
+if [[ ${#ERR_MSGS[*]} -eq 0 ]]; then
+    echo "OK: no indexes with future dates or that haven't been curated were found."
+    exit 0
+else
+    join_by " "  "${ERR_MSGS[@]}"
     exit 2
 fi
-
-echo "OK: 0 elasticsearch indexes with future dates."
diff --git a/sensu/plugins/check-mtime.rb b/sensu/plugins/check-mtime.rb
deleted file mode 100644
index 9d690ee..0000000
--- a/sensu/plugins/check-mtime.rb
+++ /dev/null
@@ -1,92 +0,0 @@
-#! /usr/bin/env ruby
-#
-#   check-mtime
-#
-# DESCRIPTION:
-#   This plugin checks a given file's modified time.
-#
-# OUTPUT:
-#   plain text
-#
-# PLATFORMS:
-#   Linux, BSD
-#
-# DEPENDENCIES:
-#   gem: sensu-plugin
-#
-# USAGE:
-#  #YELLOW
-#
-# NOTES:
-#
-# LICENSE:
-#   Copyright 2014 Sonian, Inc. and contributors. <support@sensuapp.org>
-#   Released under the same terms as Sensu (the MIT license); see LICENSE
-#   for details.
-#
-
-require 'sensu-plugin/check/cli'
-require 'fileutils'
-
-class Mtime < Sensu::Plugin::Check::CLI
-  option :file,
-         description: 'File to check last modified time',
-         short: '-f FILE',
-         long: '--file FILE'
-
-  option :warning_age,
-         description: 'Warn if mtime greater than provided age in seconds',
-         short: '-w SECONDS',
-         long: '--warning SECONDS'
-
-  option :critical_age,
-         description: 'Critical if mtime greater than provided age in seconds',
-         short: '-c SECONDS',
-         long: '--critical SECONDS'
-
-  option :ok_no_exist,
-         description: 'OK if file does not exist',
-         short: '-o',
-         long: '--ok-no-exist',
-         boolean: true,
-         default: false
-
-  option :ok_zero_size,
-         description: 'OK if file has zero size',
-         short: '-z',
-         long: '--ok-zero-size',
-         boolean: true,
-         default: false
-
-  def run_check(type, age)
-    to_check = config["#{type}_age".to_sym].to_i
-    # #YELLOW
-    if to_check > 0 && age >= to_check # rubocop:disable GuardClause
-      send(type, "file is #{age - to_check} seconds past #{type}")
-    end
-  end
-
-  def run
-    unknown 'No file specified' unless config[:file]
-    unknown 'No warn or critical age specified' unless config[:warning_age] || config[:critical_age]
-    if File.exist?(config[:file])
-      if File.size?(config[:file]).nil? && !config[:ok_zero_size]
-        critical 'file has zero size'
-      end
-    end
-    f = Dir.glob(config[:file]).first
-    if f
-      if File.size?(f).nil? && !config[:ok_zero_size]
-        critical 'file has zero size'
-      end
-      age = Time.now.to_i - File.mtime(f).to_i
-      run_check(:critical, age) || run_check(:warning, age) || ok("file is #{age} seconds old")
-    else
-      if config[:ok_no_exist]
-        ok 'file does not exist'
-      else
-        critical 'file not found'
-      end
-    end
-  end
-end
\ No newline at end of file