add sigv4 signing to client for aws es support

lib/logstash/filters/elasticsearch.rb

@@ -11,7 +11,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   # List of elasticsearch hosts to use for querying.
   config :hosts, :validate => :array,  :default => [ "localhost:9200" ]
-  
+
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
@@ -42,6 +42,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # AWS Auth - Region
+  config :region, :validate => :string
+
+  # AWS Auth - Access Key ID
+  config :aws_access_key_id, :validate => :string
+
+  # AWS Auth - Secret Access Key
+  config :aws_secret_access_key, :valdiate => :string
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -143,7 +152,10 @@ def client_options
       :ssl => @ssl,
       :hosts => @hosts,
       :ca_file => @ca_file,
-      :logger => @logger
+      :logger => @logger,
+      :aws_access_key_id => @aws_access_key_id,
+      :aws_secret_access_key => @aws_secret_access_key,
+      :region => @region
     }
   end
 

lib/logstash/filters/elasticsearch/client.rb

@@ -1,7 +1,8 @@
 # encoding: utf-8
-require "elasticsearch"
-require "base64"
-require "elasticsearch/transport/transport/http/manticore"
+require 'elasticsearch'
+require 'base64'
+require 'elasticsearch/transport/transport/http/manticore'
+require 'faraday_middleware/aws_sigv4'
 
 
 module LogStash
@@ -15,19 +16,41 @@ def initialize(user, password, options={})
         hosts   = options[:hosts]
         @logger = options[:logger]
 
+        aws_access_key_id = options[:aws_access_key_id]
+        aws_secret_access_key = options[:aws_secret_access_key]
+        region = options[:region]
+
         transport_options = {}
         if user && password
           token = ::Base64.strict_encode64("#{user}:#{password.value}")
           transport_options[:headers] = { Authorization: "Basic #{token}" }
         end
 
-        hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
+        hosts = hosts.map {|h| { host: h, scheme: 'https' } } if ssl
         # set ca_file even if ssl isn't on, since the host can be an https url
         ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
         ssl_options ||= {}
 
-        @logger.info("New ElasticSearch filter client", :hosts => hosts)
-        @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
+        if aws_access_key_id && aws_secret_access_key && region
+          @logger.info("New AWS ElasticSearch filter client", hosts: hosts)
+          @client = ::Elasticsearch::Client.new( hosts: hosts, port:443 ) do |f|
+            f.request(
+              :aws_sigv4,
+              service: 'es',
+              region: region,
+              access_key_id: aws_access_key_id,
+              secret_access_key: aws_secret_access_key
+            )
+          end
+        else
+        @logger.info("New ElasticSearch filter client", hosts: hosts)
+          @client = ::Elasticsearch::Client.new(
+            hosts: hosts,
+            transport_options: transport_options,
+            transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+            ssl: ssl_options
+          )
+        end
       end
 
       def search(params)

logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.6.0'
+  s.version         = '3.7.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'elasticsearch', ">= 5.0.3", " <6.0.0"
   s.add_runtime_dependency 'manticore', "~> 0.6"
+  s.add_runtime_dependency 'faraday_middleware-aws-sigv4'
 
   s.add_development_dependency 'logstash-devutils'
 end