Added https support

Added https support to communicate with monasca-log-api and keystone securely. The usage of https is configurable. Additionally replaced 'rest-client' library with ruby built-in library net/http. Change-Id: Ia8be223365b94923daf3eb70f0edf1d478c804d9
witekest · Jun 2, 2016 · fb8c393 · fb8c393
1 parent 96f1a90
commit fb8c393
Show file tree

Hide file tree

Showing 12 changed files with 229 additions and 223 deletions.
diff --git a/README.md b/README.md
@@ -121,9 +121,10 @@ Save the configfile wherever you like. For example ~/logstash.conf
 
 | name | description | type | required | default | example |
 | --- | --- | --- | --- | --- | --- |
-| monasca_log_api | monasca log api url | string | true | | http://192.168.10.4:8080 |
-| monasca_log_api_version | monasca log api version | string | false | v3.0 | |
-| keystone_api | keystone api url | string | true | | http://192.168.10.5:35357/v3 |
+| monasca_log_api_url | monasca log api url | string | true | | https://192.168.10.4:5607/v3.0 |
+| monasca_log_api_insecure | set to true if monasca-log-api is using an insecure ssl certificate  | boolean | false | false | |
+| keystone_api_url | keystone api url | string | true | | http://192.168.10.5:35357/v3 |
+| keystone_api_insecure | set to true if keystone is using an insecure ssl certificate | boolean | false | false | |
 | project_name | Keystone user credentials: project name | string | true | | monasca |
 | username | Keystone user credentials: username | string | true | | admin-agent |
 | password | Keystone user credentials: password | string | true | | password |
@@ -140,8 +141,8 @@ Save the configfile wherever you like. For example ~/logstash.conf
 ```bash
 output {
   monasca_log_api {
-    monasca_log_api => "http://192.168.10.4:8074"
-    keystone_api => "http://192.168.10.5:35357/v3"
+    monasca_log_api_url => "http://192.168.10.4:8074/v3.0"
+    keystone_api_url => "http://192.168.10.5:35357/v3"
     project_name => "cmm"
     username => "cmm-operator"
     password => "admin"
@@ -154,17 +155,18 @@ output {
 ```bash
 output {
   monasca_log_api {
-    monasca_log_api => "http://192.168.10.4:8074"
-    monasca_log_api_version => "v3.0"
-    keystone_api => "http://192.168.10.5:35357/v3"
+    monasca_log_api_url => "https://192.168.10.4:5607/v3.0"
+    monasca_log_api_insecure => true
+    keystone_api_url => "http://192.168.10.5:35357/v3"
+    keystone_api_insecure => false
     project_name => "cmm"
     username => "cmm-operator"
     password => "admin"
     domain_id => "default"
-    dimensions => ["hostname:kamil", "ip:10.10.10.10"]
-    num_of_logs => 100
-    delay => 1
-    elapsed_time_sec => 600
+    dimensions => ["hostname: monasca", "ip:192.168.10.4"]
+    num_of_logs => 125
+    delay => 10
+    elapsed_time_sec => 30
     max_data_size_kb => 5120
   }
 }
@@ -206,4 +208,3 @@ input {
 ## Open tasks
 * Language translations (Replace hardcoded String messages with a configuration/language file)
 * Exception handling (monasca-api requests)
-* https support
diff --git a/lib/logstash/outputs/helper/url_helper.rb b/lib/logstash/outputs/helper/url_helper.rb
diff --git a/lib/logstash/outputs/keystone/keystone_client.rb b/lib/logstash/outputs/keystone/keystone_client.rb
@@ -16,8 +16,9 @@
 
 require 'logstash/environment'
 require 'logger'
+require 'json'
+require 'net/http'
 
-require_relative '../helper/url_helper'
 require_relative './token'
 
 # This class connects to keystone and authenticates an user.
@@ -28,29 +29,33 @@ class KeystoneClient
 
       @client = nil
 
-      def initialize(host)
+      def initialize(url, insecure=false)
         @logger = Cabin::Channel.get(LogStash)
-        @client = get_client(host, '/v3/auth/tokens')
+        @uri = URI.parse(url)
+        @http = Net::HTTP.new(@uri.host, @uri.port)
+        if @uri.scheme == 'https'
+          @http.use_ssl = true
+          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if insecure
+        end
       end
 
       def authenticate(domain_id, username, password, project_name)
         auth_hash = generate_hash(domain_id, username, password, project_name)
-        response = request(auth_hash)
+        post_header = {
+              'Accept' => 'application/json',
+              'Content-Type' => 'application/json',
+          }
+        response = request('/auth/tokens', post_header, auth_hash)
         handle_response(response)
       end
 
       private
 
-      def request(auth_hash)
-        @client.post(
-          auth_hash,
-          :content_type => 'application/json',
-          :accept => 'application/json')
-      end
-
-      def get_client(host, path)
-        RestClient::Resource.new(
-          LogStash::Outputs::Helper::UrlHelper.generate_url(host, path).to_s)
+      def request(path, header, body)
+        @logger.debug('Sending authentication to ', :url => @uri.to_s)
+        post_request = Net::HTTP::Post.new(@uri.request_uri + path, header)
+        post_request.body = body
+        @http.request(post_request)
       end
 
       def generate_hash(domain_id, username, password, project_name)
@@ -77,16 +82,16 @@ def generate_hash(domain_id, username, password, project_name)
       end
 
       def handle_response(response)
-        case response.code
-        when 201
+        case response
+        when Net::HTTPCreated
           expires_at = DateTime.parse(
             JSON.parse(response.body)["token"]["expires_at"])
 
           @logger.debug("Authentication succeed: code=#{response.code}, "\
-            "auth-token=#{response.headers[:x_subject_token]}, "\
+            "auth-token=#{response['X-Subject-Token']}, "\
             "expires_at=#{expires_at.to_time}")
 
-          {:token => response.headers[:x_subject_token],
+          {:token => response['X-Subject-Token'],
             :expires_at => expires_at}
         else
           @logger.info("Authentication failed. Response=#{response}")

diff --git a/lib/logstash/outputs/keystone/token.rb b/lib/logstash/outputs/keystone/token.rb
@@ -37,9 +37,9 @@ def set_token id, expires_at
         @expires_at = expires_at
       end
 
-      def set_keystone_client keystone_api
+      def set_keystone_client(keystone_api, insecure)
         @keystone_client = LogStash::Outputs::Keystone::KeystoneClient
-          .new keystone_api
+          .new(keystone_api, insecure)
       end
 
       def initialize

diff --git a/lib/logstash/outputs/monasca/monasca_log_api_client.rb b/lib/logstash/outputs/monasca/monasca_log_api_client.rb
@@ -14,25 +14,23 @@
 
 # encoding: utf-8
 
-require 'rest-client'
 require 'logger'
-require 'singleton'
-
-# relative requirements
-require_relative '../helper/url_helper'
+require 'json'
+require 'net/http'
 
 # This class creates a connection to monasca-log-api
 module LogStash::Outputs
   module Monasca
     class MonascaLogApiClient
 
-      SUPPORTED_API_VERSION = %w(3.0)
-
-      def initialize(host, version)
+      def initialize(url, insecure=false)
         @logger = Cabin::Channel.get(LogStash)
-        rest_client_url = LogStash::Outputs::Helper::UrlHelper
-          .generate_url(host, '/' + check_version(version)).to_s
-        @rest_client = RestClient::Resource.new(rest_client_url)
+        @uri = URI.parse(url)
+        @http = Net::HTTP.new(@uri.host, @uri.port)
+        if @uri.scheme == 'https'
+          @http.use_ssl = true
+          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if insecure
+        end
       end
 
       def send_logs(logs, auth_token)
@@ -41,7 +39,8 @@ def send_logs(logs, auth_token)
               'X-Auth-Token' => auth_token,
               'Content-Type' => 'application/json',
           }
-          request(logs.to_json, post_header)
+          response = request('/logs', post_header, logs.to_json)
+          handle_response(response)
         rescue => e
           @logger.warn('Sending event to monasca-log-api threw exception',
             :exceptionew => e)
@@ -50,20 +49,21 @@ def send_logs(logs, auth_token)
 
       private
 
-      def request(body, header)
-        @logger.debug('Sending data to ', :url => @rest_client.url)
-        @rest_client['logs'].post(body, header)
+      def request(path, header, body)
+        @logger.debug('Sending data to ', :url => @uri.to_s)
+        post_request = Net::HTTP::Post.new(@uri.request_uri + path, header)
+        post_request.body = body
+        @http.request(post_request)
       end
 
-      def check_version(version)
-        tmp_version = version.sub('v','')
-
-        unless SUPPORTED_API_VERSION.include? tmp_version
-          raise "#{tmp_version} is not supported, "\
-            "supported versions are #{SUPPORTED_API_VERSION}"
+      def handle_response(response)
+        case response
+        when Net::HTTPNoContent
+          @logger.debug("Successfully sent logs")
+        else
+          # TODO: Handle logs which could not be sent
+          @logger.error("Failed to send logs. Response=#{response}")
         end
-
-        version
       end
 
     end

diff --git a/lib/logstash/outputs/monasca_log_api.rb b/lib/logstash/outputs/monasca_log_api.rb
@@ -32,12 +32,14 @@ class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
   config_name 'monasca_log_api'
 
   # monasca-log-api configuration
-  config :monasca_log_api, :validate => :string, :required => true
-  config :monasca_log_api_version, :validate => :string, :required => false,
-    :default => "v3.0"
+  config :monasca_log_api_url, :validate => :string, :required => true
+  config :monasca_log_api_insecure, :validate => :boolean, :required => false,
+    :default => false
 
   # keystone configuration
-  config :keystone_api, :validate => :string, :required => true
+  config :keystone_api_url, :validate => :string, :required => true
+  config :keystone_api_insecure, :validate => :boolean, :required => false,
+    :default => false
   config :project_name, :validate => :string, :required => true
   config :username, :validate => :string, :required => true
   config :password, :validate => :string, :required => true
@@ -64,7 +66,7 @@ def register
     @logger.info('Registering keystone user',
       :username => username, :project_name => project_name)
     @monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient
-      .new monasca_log_api, monasca_log_api_version
+      .new monasca_log_api_url, monasca_log_api_insecure
     @logs = Hash.new
     @start_time = nil
     init_token
@@ -87,13 +89,12 @@ def close
 
   def init_token
     token = LogStash::Outputs::Keystone::Token.instance
-    token.set_keystone_client(keystone_api)
+    token.set_keystone_client(keystone_api_url, keystone_api_insecure)
     check_token
   end
 
   def encode(event)
     log = generate_log_from_event(event)
-
     log_bytesize = bytesize_of(log)
     logs_bytesize = bytesize_of(@logs)
 

diff --git a/logstash-output-monasca_log_api.gemspec b/logstash-output-monasca_log_api.gemspec
@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-monasca_log_api'
-  s.version = '0.4'
+  s.version = '0.5'
   s.licenses = ['Apache License 2.0']
   s.summary = 'This gem is a logstash output plugin to connect via http to monasca-log-api.'
   s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
@@ -22,9 +22,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-core', '~> 2.0'
   s.add_runtime_dependency 'logstash-codec-plain', '~> 2.0'
   s.add_runtime_dependency 'logstash-codec-json', '~> 2.0'
-  s.add_runtime_dependency 'rest-client', '~> 1.8'
   s.add_runtime_dependency 'vine', '~> 0.2'
   s.add_development_dependency 'logstash-devutils', '~> 0.0.14'
   s.add_development_dependency 'simplecov', '~> 0.10'
   s.add_development_dependency 'rubocop', '~> 0.37.2'
+  s.add_development_dependency 'webmock', '~> 2.0'
 end
diff --git a/spec/outputs/helper/url_helper_spec.rb b/spec/outputs/helper/url_helper_spec.rb