Index Template Support Update

lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -50,6 +50,12 @@ def self.included(mod)
       # If not set, the included template will be used.
       mod.config :template, :validate => :path
 
+      # The legacy_template option will use the old /_template (ELK >7.7) 
+      # path for creating index templates.
+      # It will also construct the ilm configurations for the template in a manner
+      # which is compatible with legacy templates
+      mod.config :legacy_template, :validate => :boolean, :default => true
+
       # The template_overwrite option will always overwrite the indicated template
       # in Elasticsearch with either the one indicated by template or the included one.
       # This option is set to false by default. If you always want to stay up to date

lib/logstash/outputs/elasticsearch/http_client.rb

@@ -252,6 +252,10 @@ def http_compression
       client_settings.fetch(:http_compression, false)
     end
 
+    def legacy_template
+      client_settings.fetch(:legacy)
+    end
+
     def build_adapter(options)
       timeout = options[:timeout] || 0
 
@@ -343,11 +347,19 @@ def exists?(path, use_get=false)
     end
 
     def template_exists?(name)
-      exists?("/_template/#{name}")
+      if legacy_template
+        exists?("/_template/#{name}")
+      else
+        exists?("/_index_template/#{name}")
+      end
     end
 
     def template_put(name, template)
-      path = "_template/#{name}"
+      if legacy_template
+        path = "_template/#{name}"
+      else
+        path = "_index_template/#{name}"
+      end
       logger.info("Installing elasticsearch template to #{path}")
       @pool.put(path, nil, LogStash::Json.dump(template))
     end

lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -9,7 +9,8 @@ def self.build(logger, hosts, params)
         :pool_max_per_route => params["pool_max_per_route"],
         :check_connection_timeout => params["validate_after_inactivity"],
         :http_compression => params["http_compression"],
-        :headers => params["custom_headers"] || {}
+        :headers => params["custom_headers"] || {},
+        :legacy => params["legacy_template"]
       }
 
       client_settings[:proxy] = params["proxy"] if params["proxy"]

lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -32,14 +32,63 @@ def self.install(client, template_name, template, template_overwrite)
     end
 
     def self.add_ilm_settings_to_template(plugin, template)
-      # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
-      # definition - remove any existing definition of 'template'
-      template.delete('template') if template.include?('template')
-      template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
-      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
-        plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.")
+      if plugin.original_params.key?('ilm_rollover_alias')
+        # If no index patterns in the 'index_pattern' array would include the 'ilm_rollover_alias' add the 'ilm_rollover_alias' to the 'index_pattern' array
+	if template.key?('index_patterns') && template['index_patterns'].kind_of?(Array) && !template['index_patterns'].any? { |idx_pattern| "#{plugin.ilm_rollover_alias}-".start_with?(idx_pattern.tr('*','')) }
+          plugin.logger.info("Adding index pattern name for rollover alias as ILM is enabled.")
+  	  template['index_patterns'].append("#{plugin.ilm_rollover_alias}-*")
+        # If 'index_pattern' is not an array and doesn't include the 'ilm_rollover_alias' set the 'index_pattern' to the 'ilm_rollover_alias'
+  	elsif template.key?('index_patterns') && !template['index_patterns'].kind_of?(Array) && !"#{plugin.ilm_rollover_alias}-".start_with?(template['index_patterns'].tr('*',''))
+          plugin.logger.info("Overwriting index pattern name for rollover alias as ILM is enabled.")
+ 	  template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
+        # If 'index_pattern' doesn't exist, set it to the 'ilm_rollover_alias'
+	elsif !template.key?('index_patterns')
+          plugin.logger.info("Setting index pattern name for rollover alias as ILM is enabled.")
+	  template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
+	end
+      end
+      if plugin.legacy_template
+        # definition - remove any existing definition of 'template'
+        template.delete('template') if template.include?('template')
+        # Create settings hash if not in the template, but ilm is enabled
+        if !template.key?('settings')
+          template['settings'] = { }
+        end
+        if plugin.original_params.key?('ilm_rollover_alias')
+          if template['settings']
+	    # Overwrite the rollover_alias, sense it was defined in the output plugin
+            plugin.logger.info("Overwriting index rollover alias with plugin defined alias.")
+            template['settings'].update({ 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
+          end
+        end
+        if plugin.original_params.key?('ilm_policy')
+          if template['settings'] 
+	    # Overwrite the ilm_policy, sense it was defined in the output plugin
+            plugin.logger.info("Overwriting index lifecycle name with plugin defined ilm policy.")
+            template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy})
+	  end
+        end
+      else
+        #plugin.logger.warn(plugin.original_params.key?('ilm_rollover_alias'))
+        # Create settings hash if not in the template, but ilm is enabled
+        if !template['template'].key?('settings')
+          template['template']['settings'] = { }
+        end
+        if plugin.original_params.key?('ilm_rollover_alias')
+          if template['template']['settings']
+	    # Overwrite the rollover_alias, sense it was defined in the output plugin
+            plugin.logger.info("Overwriting index rollover alias with plugin defined alias.")
+            template['template']['settings'].update({ 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
+          end
+        end
+        if plugin.original_params.key?('ilm_policy')
+          if template['template']['settings'] 
+	    # Overwrite the ilm_policy, sense it was defined in the output plugin
+            plugin.logger.info("Overwriting index lifecycle name with plugin defined ilm policy.")
+            template['template']['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy})
+	  end
+        end
       end
-      template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
     end
 
     # Template name - if template_name set, use it