Completely rewrote indexing: bulk, dynamic names.

Bulk indexing is now used when inserting documents. Dynamic indexes are created based on the UTC date (statsd-2014.03.03). A sample index template is provided.
markkimsal · Mar 3, 2014 · 1003587 · 1003587
1 parent 702efcd
commit 1003587
Show file tree

Hide file tree

Showing 4 changed files with 304 additions and 100 deletions.
diff --git a/README.md b/README.md
@@ -19,16 +19,48 @@ Originally written by Github user rameshpy, this library was created as a featur
 ## Configuration
 
 Merge the following configuration into your top-level existing configuration.
+Add a structure to your configuration called "elasticsearch"
 
+```js
+
+ elasticsearch: {
+	 port:          9200,
+	 host:          "localhost",
+	 indexPrefix:   "statsd",
+	 countType:     "counter",
+	 timerType:     "timer"
+	 timerDataType: "timer_data"
+ }
+```
+
+The field _indexPrefix_ is used as the prefix for your dynamic indices: for example "statsd-2014.02.04"
+
+The type configuration options allow you to specify different elasticsearch _types for each statsd measurement.
+
+To configure Elasticsearch to automatically apply index template settings based on a naming pattern look at the es-index-template.sh file.  It will probably need customization (the timer_data type) for your particular statsd configuration (re: threshold pct).
+
+## Metric Name Mapping
+
+Each key sent to the elasticsearch backend will be broken up by dots (.) and each part of the key will be treated as a document property in elastic search.
+For example:
+
+```js
+accounts.authentication.password.failed:1|c
+```
+
+The above would be mapped into a JSON document like this:
 ```js
 {
-   elasticPort: 9200
- , elasticHost: "localhost"
- , elasticFlushInterval: 10000
- , elasticIndex: "statsd"
- , elasticIndexType: "stats"
- , backends: ['statsd-elasticsearch-backend']
+	"_type":"counter",
+	"ns":"accounts",
+	"grp":"authentication",
+	"tgt":"password",
+	"act":"failed",
+	"val":"1",
+	"@timestamp":"1393853783000"
 }
 ```
 
-The field _elasticIndex_ is used as the prefix for your dynamic indices: for example "statsd-2014.02.04"
+Currently the values are hardcoded to: namespace, group, target, and action, as in the above example.  Having configurable naming conventions is the goal of a 1.0 release.
+The idea for mapping came mostly from: [http://matt.aimonetti.net/posts/2013/06/26/practical-guide-to-graphite-monitoring/]
+
diff --git a/es-index-template.sh b/es-index-template.sh
@@ -0,0 +1,144 @@
+curl -XPUT localhost:9200/_template/statsd-template -d '
+{
+    "template" : "statsd-*",
+    "settings" : {
+        "number_of_shards" : 1
+    },
+    "mappings" : {
+        "counter" : {
+            "_source" : { "enabled" : true },
+            "properties": {
+                "@timestamp": {
+                    "format": "dateOptionalTime",
+                    "type": "date"
+                },
+                "val": {
+                    "type": "long",
+                    "index": "not_analyzed"
+                },
+                "ns": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "grp": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "tgt": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "act": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                }
+            }
+        },
+        "timer" : {
+            "_source" : { "enabled" : true },
+            "properties": {
+                "@timestamp": {
+                    "format": "dateOptionalTime",
+                    "type": "date"
+                },
+                "val": {
+                    "type": "long",
+                    "index": "not_analyzed"
+                },
+                "ns": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "grp": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "tgt": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "act": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                }
+            }
+        },
+        "timer_data" : {
+            "_source" : { "enabled" : true },
+            "properties": {
+                "@timestamp": {
+                    "format": "dateOptionalTime",
+                    "type": "date"
+                },
+                "count_ps": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "count": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "upper": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "lower": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "mean": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "median": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "mean": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "upper": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "std": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                 "sum": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "mean_90": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "upper_90": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "sum_90": {
+                    "type": "float",
+                    "index": "not_analyzed"
+                },
+                "ns": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "grp": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "tgt": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                },
+                "act": {
+                    "type": "string",
+                    "index": "not_analyzed"
+                }
+            }
+        }
+    }
+}'