api: add client authentication (BROKEN)

config/logstash.yml

@@ -118,26 +118,39 @@
 #
 # config.support_escapes: false
 #
-# ------------ HTTP API Settings -------------
+# ------------ API Settings -------------
 # Define settings related to the HTTP API here.
 #
 # The HTTP API is enabled by default. It can be disabled, but features that rely
 # on it will not work as intended.
-# http.enabled: true
+#
+# api.enabled: true
 #
 # By default, the HTTP API is bound to only the host's local loopback interface,
 # ensuring that it is not accessible to the rest of the network. Because the API
 # includes neither authentication nor authorization and has not been hardened or
 # tested for use as a publicly-reachable API, binding to publicly accessible IPs
 # should be avoided where possible.
 #
-# http.host: 127.0.0.1
+# api.http.host: 127.0.0.1
 #
 # The HTTP API web server will listen on an available port from the given range.
 # Values can be specified as a single port (e.g., `9600`), or an inclusive range
 # of ports (e.g., `9600-9700`).
 #
-# http.port: 9600-9700
+# api.http.port: 9600-9700
+#
+# The HTTP API includes a customizable "environment" value in its response,
+# which can be configured here.
+#
+# api.environment: "production"
+#
+# The HTTP API can be secured with SSL (TLS). To do so, you will need to provide
+# the path to a password-protected keystore in jks format, along with credentials.
+#
+# api.ssl.enabled: false
+# api.ssl.keystore.path: /path/to/keystore.jks
+# api.ssl.keystore.password: "y0uRp4$$w0rD"
 #
 # ------------ Module Settings ---------------
 # Define modules here.  Modules definitions must be defined as an array.
@@ -240,17 +253,6 @@
 #
 # path.dead_letter_queue:
 #
-# ------------ Metrics Settings --------------
-#
-# Bind address for the metrics REST endpoint
-#
-# http.host: "127.0.0.1"
-#
-# Bind port for the metrics REST endpoint, this option also accept a range
-# (9600-9700) and logstash will pick up the first available ports.
-#
-# http.port: 9600-9700
-#
 # ------------ Debugging Settings --------------
 #
 # Options for log.level:

docker/data/logstash/env2yaml/env2yaml.go

@@ -48,6 +48,10 @@ func squashSetting(setting string) string {
 // return the canonical setting name. eg. 'pipeline.unsafe_shutdown'
 func normalizeSetting(setting string) (string, error) {
 	valid_settings := []string{
+		"api.enabled",
+		"api.http.host",
+		"api.http.port",
+		"api.environment",
 		"node.name",
 		"path.data",
 		"pipeline.id",
@@ -80,9 +84,10 @@ func normalizeSetting(setting string) (string, error) {
 		"dead_letter_queue.max_bytes",
 		"dead_letter_queue.flush_interval",
 		"path.dead_letter_queue",
-		"http.host",
-		"http.port",
-		"http.enabled",
+		"http.enabled",     // DEPRECATED: prefer `api.enabled`
+		"http.environment", // DEPRECATED: prefer `api.environment`
+		"http.host",        // DEPRECATED: prefer `api.http.host`
+		"http.port",        // DEPRECATED: prefer `api.http.port`
 		"log.level",
 		"log.format",
 		"modules",

docs/static/logging.asciidoc

@@ -55,7 +55,7 @@ downtime. Instead, you can dynamically update logging levels through the logging
 immediately and do not need a restart. 
 
 NOTE: By default, the logging API attempts to bind to `tcp:9600`. If this port is already in use by another Logstash
-instance, you need to launch Logstash with the `--http.port` flag specified to bind to a different port. See
+instance, you need to launch Logstash with the `--api.http.port` flag specified to bind to a different port. See
 <<command-line-flags>> for more information.
 
 ===== Retrieve list of logging configurations

docs/static/monitoring/monitoring-apis.asciidoc

@@ -31,7 +31,7 @@ Example response:
 --------------------------------------------------
 
 NOTE: By default, the monitoring API attempts to bind to `tcp:9600`. If this port is already in use by another Logstash
-instance, you need to launch Logstash with the `--http.port` flag specified to bind to a different port. See
+instance, you need to launch Logstash with the `--api.http.port` flag specified to bind to a different port. See
 <<command-line-flags>> for more information.
 
 [discrete]

docs/static/running-logstash-command-line.asciidoc

@@ -207,10 +207,13 @@ With this command, Logstash concatenates three config files, `/tmp/one`, `/tmp/t
   How frequently to poll the configuration location for changes. The default value is "3s".
   Note that the unit qualifier (`s`) is required.
 
-*`--http.host HTTP_HOST`*::
+*`--api-enabled ENABLED`*::
+  The HTTP API is enabled by default, but can be disabled by passing `false` to this option.
+
+*`--api.http.host HTTP_HOST`*::
   Web API binding host. This option specifies the bind address for the metrics REST endpoint. The default is "127.0.0.1".
 
-*`--http.port HTTP_PORT`*::
+*`--api.http.port HTTP_PORT`*::
   Web API http port. This option specifies the bind port for the metrics REST endpoint. The default is 9600-9700.
   This setting accepts a range of the format 9600-9700. Logstash will pick up the first available port.
 

docs/static/settings-file.asciidoc

@@ -235,13 +235,21 @@ Values other than `disabled` are currently considered BETA, and may produce unin
 | The directory path where the data files will be stored for the dead-letter queue.
 | `path.data/dead_letter_queue`
 
-| `http.host`
-| The bind address for the metrics REST endpoint.
+| `api.enabled`
+|  The HTTP API is enabled by default. It can be disabled, but features that rely on it will not work as intended.
+| `true`
+
+| `api.environment`
+| The API returns the provided string as a part of its response. Setting your environment may help to disambiguate between similarly-named nodes in production vs test environments.
+| `production`
+
+| `api.http.host`
+| The bind address for the HTTP API endpoint.
 | `"127.0.0.1"`
 
-| `http.port`
-| The bind port for the metrics REST endpoint.
-| `9600`
+| `api.http.port`
+| The bind port for the HTTP API endpoint.
+| `9600-9700`
 
 | `log.level`
 a|

logstash-core/lib/logstash/agent.rb

@@ -62,9 +62,6 @@ def initialize(settings = LogStash::SETTINGS, source_loader = nil)
     @pipelines_registry = LogStash::PipelinesRegistry.new
 
     @name = setting("node.name")
-    @http_host = setting("http.host")
-    @http_port = setting("http.port")
-    @http_environment = setting("http.environment")
     # Generate / load the persistent uuid
     id
 
@@ -427,17 +424,16 @@ def dispatch_events(converge_results)
   end
 
   def start_webserver_if_enabled
-    if @settings.get_value("http.enabled")
+    if @settings.get_value("api.enabled")
       start_webserver
     else
-      @logger.info("HTTP API is disabled (`http.enabled=false`); webserver will not be started.")
+      @logger.info("HTTP API is disabled (`api.enabled=false`); webserver will not be started.")
     end
   end
 
   def start_webserver
     @webserver_control_lock.synchronize do
-      options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
-      @webserver = LogStash::WebServer.new(@logger, self, options)
+      @webserver = LogStash::WebServer.from_settings(@logger, self, settings)
       @webserver_thread = Thread.new(@webserver) do |webserver|
         LogStash::Util.set_thread_name("Api Webserver")
         webserver.run

logstash-core/lib/logstash/environment.rb

@@ -69,10 +69,14 @@ module Environment
            Setting::Boolean.new("help", false),
             Setting::Boolean.new("enable-local-plugin-development", false),
             Setting::String.new("log.format", "plain", true, ["json", "plain"]),
-           Setting::Boolean.new("http.enabled", true),
-            Setting::String.new("http.host", "127.0.0.1"),
-         Setting::PortRange.new("http.port", 9600..9700),
-            Setting::String.new("http.environment", "production"),
+           Setting::Boolean.new("api.enabled", true).with_deprecated_alias("http.enabled"),
+            Setting::String.new("api.http.host", "127.0.0.1").with_deprecated_alias("http.host"),
+         Setting::PortRange.new("api.http.port", 9600..9700).with_deprecated_alias("http.port"),
+            Setting::String.new("api.environment", "production").with_deprecated_alias("http.environment"),
+           Setting::Boolean.new("api.ssl.enabled", false),
+  Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable,
+          Setting::Password.new("api.ssl.keystore.password", nil, false).nullable,
+            Setting::String.new("api.ssl.client_authentication", "none", true, %w(none optional required)),
             Setting::String.new("queue.type", "memory", true, ["persisted", "memory"]),
             Setting::Boolean.new("queue.drain", false),
             Setting::Bytes.new("queue.page_capacity", "64mb"),
@@ -119,6 +123,15 @@ module Environment
     end
   end
 
+  SETTINGS.on_post_process do |settings|
+    if settings.get('api.ssl.enabled') && !settings.set?('api.ssl.keystore.path')
+      raise ArgumentError.new('Setting `api.ssl.enabled` is true, but required `api.ssl.keystore.path` is not provided. Please provide a valid keystore in `logstash.yml`')
+    end
+    if settings.set?('api.ssl.keystore.path') && !settings.set?('api.ssl.keystore.password')
+      raise ArgumentError.new("Setting `api.ssl.keystore.path` provided without required `api.ssl.keystore.password`. Please provided credentials in `logstash.yml`")
+    end
+  end
+
   module Environment
     extend self
 

logstash-core/lib/logstash/patches/clamp.rb

@@ -78,8 +78,17 @@ def define_deprecated_accessors_for(option, opts, &block)
 
       def define_deprecated_writer_for(option, opts, &block)
         define_method(option.write_method) do |value|
-          LogStash::DeprecationMessage.instance << "DEPRECATION WARNING: The flag #{option.switches} has been deprecated, please use \"--#{opts[:new_flag]}=#{opts[:new_value]}\" instead."
-          LogStash::SETTINGS.set(opts[:new_flag], opts[:new_value])
+          new_flag = opts[:new_flag]
+          new_value = opts.fetch(:new_value, value)
+          passthrough = opts.fetch(:passthrough, false)
+
+          LogStash::DeprecationMessage.instance << "DEPRECATION WARNING: The flag #{option.switches} has been deprecated, please use \"--#{new_flag}=#{new_value}\" instead."
+
+          if passthrough
+            LogStash::SETTINGS.set(option.attribute_name, value)
+          else
+            LogStash::SETTINGS.set(opts[:new_flag], opts.include?(:new_value) ? opts[:new_value] : value)
+          end
         end
       end
     end

logstash-core/lib/logstash/patches/puma.rb

@@ -48,6 +48,14 @@ def puts(str)
     alias_method :<<, :puts
   end
 
+  # ::Puma::Events#error(str) sends Kernel#exit
+  # let's raise something sensible instead.
+  UnrecoverablePumaError = Class.new(RuntimeError)
+  class NonCrashingPumaEvents < ::Puma::Events
+    def error(str)
+      raise UnrecoverablePumaError.new(str)
+    end
+  end
 end
 
 # Reopen the puma class to create a scoped STDERR and STDOUT

logstash-core/lib/logstash/runner.rb

@@ -205,20 +205,20 @@ class LogStash::Runner < Clamp::StrictCommand
     :attribute_name => "config.reload.interval",
     :default => LogStash::SETTINGS.get_default("config.reload.interval")
 
-  option ["--http.enabled"], "ENABLED",
-         I18n.t("logstash.runner.flag.http_enabled"),
-         :attribute_name => 'http.enabled',
-         :default => LogStash::SETTINGS.get_default('http.enabled')
+  option ["--api.enabled"], "ENABLED",
+    I18n.t("logstash.runner.flag.api_enabled"),
+    :attribute_name => 'api.enabled',
+    :default => LogStash::SETTINGS.get_default('api.enabled')
 
-  option ["--http.host"], "HTTP_HOST",
-    I18n.t("logstash.runner.flag.http_host"),
-    :attribute_name => "http.host",
-    :default => LogStash::SETTINGS.get_default("http.host")
+  option ["--api.http.host"], "HTTP_HOST",
+    I18n.t("logstash.runner.flag.api_http_host"),
+    :attribute_name => "api.http.host",
+    :default => LogStash::SETTINGS.get_default("api.http.host")
 
-  option ["--http.port"], "HTTP_PORT",
-    I18n.t("logstash.runner.flag.http_port"),
-    :attribute_name => "http.port",
-    :default => LogStash::SETTINGS.get_default("http.port")
+  option ["--api.http.port"], "HTTP_PORT",
+    I18n.t("logstash.runner.flag.api_http_port"),
+    :attribute_name => "api.http.port",
+    :default => LogStash::SETTINGS.get_default("api.http.port")
 
   option ["--log.format"], "FORMAT",
     I18n.t("logstash.runner.flag.log_format"),
@@ -243,6 +243,18 @@ class LogStash::Runner < Clamp::StrictCommand
     I18n.t("logstash.runner.flag.quiet"),
     :new_flag => "log.level", :new_value => "error"
 
+  deprecated_option ["--http.enabled"], :flag,
+    I18n.t("logstash.runner.flag.http_enabled"),
+    :new_flag => "api.enabled", :passthrough => true # use settings to disambiguate
+
+  deprecated_option ["--http.host"], "HTTP_HOST",
+    I18n.t("logstash.runner.flag.http_host"),
+    :new_flag => "api.http.host", :passthrough => true # use settings to disambiguate
+
+  deprecated_option ["--http.port"], "HTTP_PORT",
+    I18n.t("logstash.runner.flag.http_port"),
+    :new_flag => "api.http.port", :passthrough => true # use settings to disambiguate
+
   # We configure the registry and load any plugin that can register hooks
   # with logstash, this needs to be done before any operation.
   SYSTEM_SETTINGS = LogStash::SETTINGS.clone
@@ -319,7 +331,7 @@ def execute
     @dispatcher = LogStash::EventDispatcher.new(self)
     LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
 
-    @settings.validate_all
+    validate_settings! or return 1
     @dispatcher.fire(:before_bootstrap_checks)
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
@@ -451,6 +463,14 @@ def log_configuration_contains_javascript_usage?
      (log_config =~ /^[^#]+script\.language\s*=\s*JavaScript/) != nil
   end
 
+  def validate_settings!
+    @settings.validate_all
+    true
+  rescue => e
+    $stderr.puts(I18n.t("logstash.runner.invalid-settings", :error => e.message))
+    return false
+  end
+
   def show_version
     show_version_logstash