diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index 3032153..687d71f 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -11,7 +11,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # List of elasticsearch hosts to use for querying. config :hosts, :validate => :array, :default => [ "localhost:9200" ] - + # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. # Field substitution (e.g. `index-name-%{date_field}`) is available config :index, :validate => :string, :default => "" @@ -42,6 +42,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # Basic Auth - password config :password, :validate => :password + # AWS Auth - Region + config :region, :validate => :string + + # AWS Auth - Access Key ID + config :aws_access_key_id, :validate => :string + + # AWS Auth - Secret Access Key + config :aws_secret_access_key, :valdiate => :string + # SSL config :ssl, :validate => :boolean, :default => false @@ -143,7 +152,10 @@ def client_options :ssl => @ssl, :hosts => @hosts, :ca_file => @ca_file, - :logger => @logger + :logger => @logger, + :aws_access_key_id => @aws_access_key_id, + :aws_secret_access_key => @aws_secret_access_key, + :region => @region } end diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index 986e797..17a7391 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -1,7 +1,8 @@ # encoding: utf-8 -require "elasticsearch" -require "base64" -require "elasticsearch/transport/transport/http/manticore" +require 'elasticsearch' +require 'base64' +require 'elasticsearch/transport/transport/http/manticore' +require 'faraday_middleware/aws_sigv4' module LogStash @@ -15,19 +16,41 @@ def initialize(user, password, options={}) hosts = options[:hosts] @logger = options[:logger] + aws_access_key_id = options[:aws_access_key_id] + aws_secret_access_key = options[:aws_secret_access_key] + region = options[:region] + transport_options = {} if user && password token = ::Base64.strict_encode64("#{user}:#{password.value}") transport_options[:headers] = { Authorization: "Basic #{token}" } end - hosts.map! {|h| { host: h, scheme: 'https' } } if ssl + hosts = hosts.map {|h| { host: h, scheme: 'https' } } if ssl # set ca_file even if ssl isn't on, since the host can be an https url ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file] ssl_options ||= {} - @logger.info("New ElasticSearch filter client", :hosts => hosts) - @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options) + if aws_access_key_id && aws_secret_access_key && region + @logger.info("New AWS ElasticSearch filter client", hosts: hosts) + @client = ::Elasticsearch::Client.new( hosts: hosts, port:443 ) do |f| + f.request( + :aws_sigv4, + service: 'es', + region: region, + access_key_id: aws_access_key_id, + secret_access_key: aws_secret_access_key + ) + end + else + @logger.info("New ElasticSearch filter client", hosts: hosts) + @client = ::Elasticsearch::Client.new( + hosts: hosts, + transport_options: transport_options, + transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, + ssl: ssl_options + ) + end end def search(params) diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index dc81a7e..610b43e 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '3.6.0' + s.version = '3.7.0' s.licenses = ['Apache License (2.0)'] s.summary = "Copies fields from previous log events in Elasticsearch to current events " s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -23,6 +23,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'elasticsearch', ">= 5.0.3", " <6.0.0" s.add_runtime_dependency 'manticore', "~> 0.6" + s.add_runtime_dependency 'faraday_middleware-aws-sigv4' s.add_development_dependency 'logstash-devutils' end