From b9ab8315046186096574f4c490f61d3fd7a07960 Mon Sep 17 00:00:00 2001 From: Julian Scheid Date: Fri, 19 Dec 2014 12:59:21 +0100 Subject: [PATCH 1/5] Ensure default tag is within size limits RFC 3164 section 4.1.3 specifies that the tag MUST NOT exceed 32 characters. This is enforced by https://github.com/eric/syslog_protocol/blob/001000ffe27a4557c3ec312b9c3c50385e6a923b/lib/syslog_protocol/packet.rb#L48-L50 Ensure that the default tag generated from the program name and PID is within these bounds. --- lib/remote_syslog_logger/udp_sender.rb | 8 +++++++- test/test_remote_syslog_logger.rb | 21 ++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/remote_syslog_logger/udp_sender.rb b/lib/remote_syslog_logger/udp_sender.rb index 867de4a..1f2d3a5 100644 --- a/lib/remote_syslog_logger/udp_sender.rb +++ b/lib/remote_syslog_logger/udp_sender.rb @@ -17,7 +17,13 @@ def initialize(remote_hostname, remote_port, options = {}) @packet.facility = options[:facility] || 'user' @packet.severity = options[:severity] || 'notice' - @packet.tag = options[:program] || "#{File.basename($0)}[#{$$}]" + @packet.tag = options[:program] || default_tag + end + + def default_tag + pid_suffix = "[#{$$}]" + max_basename_size = 32 - pid_suffix.size + "#{File.basename($0)}"[0...max_basename_size].gsub(/[^\x21-\x7E]/, '_') + pid_suffix end def transmit(message) diff --git a/test/test_remote_syslog_logger.rb b/test/test_remote_syslog_logger.rb index 8ad191e..737e90a 100644 --- a/test/test_remote_syslog_logger.rb +++ b/test/test_remote_syslog_logger.rb @@ -25,4 +25,23 @@ def test_logger_multiline message, addr = *@socket.recvfrom(1024) assert_match /This is the second line/, message end -end \ No newline at end of file + + def test_logger_default_tag + $0 = 'foo' + logger = RemoteSyslogLogger.new('127.0.0.1', @server_port) + logger.info "" + + message, addr = *@socket.recvfrom(1024) + assert_match "foo[#{$$}]: I,", message + end + + def test_logger_long_default_tag + $0 = 'x' * 64 + pid_suffix = "[#{$$}]" + logger = RemoteSyslogLogger.new('127.0.0.1', @server_port) + logger.info "" + + message, addr = *@socket.recvfrom(1024) + assert_match 'x' * (32 - pid_suffix.size) + pid_suffix + ': I,', message + end +end From cfe35a8c407c83e2810ce482b97c7c0c84b6d6e9 Mon Sep 17 00:00:00 2001 From: Julian Scheid Date: Fri, 19 Dec 2014 13:05:49 +0100 Subject: [PATCH 2/5] Ensure log messages aren't truncated RFC 3164 section 4.1 (also 4.3.2 and 6.1) specifies that the packet length MUST NOT exceed 1024 bytes. This is enforced by https://github.com/eric/syslog_protocol/blob/001000ffe27a4557c3ec312b9c3c50385e6a923b/lib/syslog_protocol/packet.rb#L16-L21 by truncating the packet to this length (by default). Ensure that we don't lose parts of long log messages by splitting them into chunks accordingly and sending each chunk in an individual packet. Prefix continuation chunks with a configurable prefix (default: ellipsis followed by space) so they can be identified as such more easily. Also make the maximum packet size configurable; while RFC 3164 specifies a maximum packet size of 1024, some implementations (e.g. rsyslog) support larger sizes. --- lib/remote_syslog_logger/udp_sender.rb | 16 +++++- test/test_remote_syslog_logger.rb | 80 ++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/lib/remote_syslog_logger/udp_sender.rb b/lib/remote_syslog_logger/udp_sender.rb index 1f2d3a5..74ce328 100644 --- a/lib/remote_syslog_logger/udp_sender.rb +++ b/lib/remote_syslog_logger/udp_sender.rb @@ -7,7 +7,9 @@ def initialize(remote_hostname, remote_port, options = {}) @remote_hostname = remote_hostname @remote_port = remote_port @whinyerrors = options[:whinyerrors] - + @max_packet_size = options[:max_packet_size] || 1024 + @continuation_prefix = options[:continuation_prefix] || '... ' + @socket = UDPSocket.new @packet = SyslogProtocol::Packet.new @@ -31,8 +33,16 @@ def transmit(message) begin next if line =~ /^\s*$/ packet = @packet.dup - packet.content = line - @socket.send(packet.assemble, 0, @remote_hostname, @remote_port) + max_content_size = @max_packet_size - packet.assemble(@max_packet_size).size + offset = 0 + line_prefix = '' + while offset < line.size + chunk_size = max_content_size - line_prefix.size + packet.content = line_prefix + line[offset...offset+chunk_size] + @socket.send(packet.assemble(@max_packet_size), 0, @remote_hostname, @remote_port) + offset += chunk_size + line_prefix = @continuation_prefix + end rescue $stderr.puts "#{self.class} error: #{$!.class}: #{$!}\nOriginal message: #{line}" raise if @whinyerrors diff --git a/test/test_remote_syslog_logger.rb b/test/test_remote_syslog_logger.rb index 737e90a..65f6fd9 100644 --- a/test/test_remote_syslog_logger.rb +++ b/test/test_remote_syslog_logger.rb @@ -44,4 +44,84 @@ def test_logger_long_default_tag message, addr = *@socket.recvfrom(1024) assert_match 'x' * (32 - pid_suffix.size) + pid_suffix + ': I,', message end + + TEST_TAG = 'foo' + TEST_HOSTNAME = 'bar' + TEST_FACILITY = 'user' + TEST_SEVERITY = 'notice' + TEST_MESSAGE = "abcdefg" * 512 + + def test_logger_long_message + _test_msg_splitting_with( + tag: TEST_TAG, + hostname: TEST_HOSTNAME, + severity: TEST_SEVERITY, + facility: TEST_FACILITY, + message: TEST_MESSAGE, + max_packet_size: nil, + continuation_prefix: nil) + end + + def test_logger_long_message_custom_packet_size + _test_msg_splitting_with( + tag: TEST_TAG, + hostname: TEST_HOSTNAME, + severity: TEST_SEVERITY, + facility: TEST_FACILITY, + message: TEST_MESSAGE, + max_packet_size: 2048, + continuation_prefix: nil) + end + + def test_logger_long_message_custom_continuation + _test_msg_splitting_with( + tag: TEST_TAG, + hostname: TEST_HOSTNAME, + severity: TEST_SEVERITY, + facility: TEST_FACILITY, + message: TEST_MESSAGE, + max_packet_size: nil, + continuation_prefix: 'frobnicate') + end + + private + + class MessageOnlyFormatter < ::Logger::Formatter + def call(severity, timestamp, progname, msg) + msg + end + end + + def _test_msg_splitting_with(options) + logger = RemoteSyslogLogger.new('127.0.0.1', @server_port, + program: options[:tag], + local_hostname: options[:hostname], + severity: options[:severity], + facility: options[:facility], + max_packet_size: options[:max_packet_size], + continuation_prefix: options[:continuation_prefix]) + logger.formatter = MessageOnlyFormatter.new + logger.info options[:message] + + packet_size = options[:max_packet_size] || 1024 + continuation_prefix = options[:continuation_prefix] || '... ' + + test_packet = SyslogProtocol::Packet.new + test_packet.hostname = options[:hostname] + test_packet.tag = options[:tag] + test_packet.severity = options[:severity] + test_packet.facility = options[:facility] + test_packet.content = '' + max_content_size = packet_size - test_packet.assemble.size + + offset = 0 + line_prefix = '' + while offset < options[:message].size + chunk_size = max_content_size - line_prefix.size + message, addr = *@socket.recvfrom(packet_size * 2) + assert_match Regexp.new(': ' + line_prefix + Regexp.escape(options[:message][offset...offset + chunk_size]) + '$'), message + offset += chunk_size + line_prefix = continuation_prefix + end + end end From f58c14e9a47eceb7b3be2cb22a4cfda7375c5001 Mon Sep 17 00:00:00 2001 From: Julian Scheid Date: Thu, 6 Oct 2016 15:19:18 +0200 Subject: [PATCH 3/5] Update test infrastructure --- Rakefile | 2 +- remote_syslog_logger.gemspec | 3 +++ test/helper.rb | 2 +- test/test_remote_syslog_logger.rb | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Rakefile b/Rakefile index d4f7da8..5c38b7c 100644 --- a/Rakefile +++ b/Rakefile @@ -60,7 +60,7 @@ task :coverage do sh "open coverage/index.html" end -require 'rake/rdoctask' +require 'rdoc/task' Rake::RDocTask.new do |rdoc| rdoc.rdoc_dir = 'rdoc' rdoc.title = "#{name} #{version}" diff --git a/remote_syslog_logger.gemspec b/remote_syslog_logger.gemspec index 6f97359..ffd8256 100644 --- a/remote_syslog_logger.gemspec +++ b/remote_syslog_logger.gemspec @@ -50,6 +50,9 @@ Gem::Specification.new do |s| ## that are needed for an end user to actually USE your code. s.add_dependency('syslog_protocol') + s.add_development_dependency('rake') + s.add_development_dependency('test-unit') + ## List your development dependencies here. Development dependencies are ## those that are only needed during development # s.add_development_dependency('DEVDEPNAME', [">= 1.1.0", "< 2.0.0"]) diff --git a/test/helper.rb b/test/helper.rb index 1198bab..6dce6f3 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -9,4 +9,4 @@ require 'remote_syslog_logger' -require 'test/unit' \ No newline at end of file +require 'minitest/autorun' diff --git a/test/test_remote_syslog_logger.rb b/test/test_remote_syslog_logger.rb index 65f6fd9..c9ddb7f 100644 --- a/test/test_remote_syslog_logger.rb +++ b/test/test_remote_syslog_logger.rb @@ -1,6 +1,6 @@ require File.expand_path('../helper', __FILE__) -class TestRemoteSyslogLogger < Test::Unit::TestCase +class TestRemoteSyslogLogger < MiniTest::Test def setup @server_port = rand(50000) + 1024 @socket = UDPSocket.new From 366dce8ee5e081013ce8abb5ae59e1db8dd7ac3c Mon Sep 17 00:00:00 2001 From: Julian Scheid Date: Thu, 6 Oct 2016 15:39:27 +0200 Subject: [PATCH 4/5] Fix chunking of UTF-8 strings --- lib/remote_syslog_logger/limit_bytesize.rb | 18 +++++++++ lib/remote_syslog_logger/udp_sender.rb | 12 +++--- remote_syslog_logger.gemspec | 1 + test/test_remote_syslog_logger.rb | 47 ++++++++++++++++++---- 4 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 lib/remote_syslog_logger/limit_bytesize.rb diff --git a/lib/remote_syslog_logger/limit_bytesize.rb b/lib/remote_syslog_logger/limit_bytesize.rb new file mode 100644 index 0000000..78f28e3 --- /dev/null +++ b/lib/remote_syslog_logger/limit_bytesize.rb @@ -0,0 +1,18 @@ +# Adapted from http://stackoverflow.com/a/12536366/2778142 +def limit_bytesize(str, size) + # Change to canonical unicode form (compose any decomposed characters). + # Works only if you're using active_support + str = str.mb_chars.compose.to_s if str.respond_to?(:mb_chars) + + # Start with a string of the correct byte size, but + # with a possibly incomplete char at the end. + new_str = str.byteslice(0, size) + + # We need to force_encoding from utf-8 to utf-8 so ruby will re-validate + # (idea from halfelf). + until new_str[-1].force_encoding(new_str.encoding).valid_encoding? + # remove the invalid char + new_str = new_str.slice(0..-2) + end + new_str +end diff --git a/lib/remote_syslog_logger/udp_sender.rb b/lib/remote_syslog_logger/udp_sender.rb index 74ce328..fa86532 100644 --- a/lib/remote_syslog_logger/udp_sender.rb +++ b/lib/remote_syslog_logger/udp_sender.rb @@ -1,5 +1,6 @@ require 'socket' require 'syslog_protocol' +require File.expand_path('../limit_bytesize', __FILE__) module RemoteSyslogLogger class UdpSender @@ -34,13 +35,14 @@ def transmit(message) next if line =~ /^\s*$/ packet = @packet.dup max_content_size = @max_packet_size - packet.assemble(@max_packet_size).size - offset = 0 line_prefix = '' - while offset < line.size - chunk_size = max_content_size - line_prefix.size - packet.content = line_prefix + line[offset...offset+chunk_size] + remaining_line = line + until remaining_line.empty? + chunk_byte_size = max_content_size - line_prefix.bytesize + chunk = limit_bytesize(remaining_line, chunk_byte_size) + packet.content = line_prefix + chunk @socket.send(packet.assemble(@max_packet_size), 0, @remote_hostname, @remote_port) - offset += chunk_size + remaining_line = remaining_line[chunk.size..-1] line_prefix = @continuation_prefix end rescue diff --git a/remote_syslog_logger.gemspec b/remote_syslog_logger.gemspec index ffd8256..fecef05 100644 --- a/remote_syslog_logger.gemspec +++ b/remote_syslog_logger.gemspec @@ -49,6 +49,7 @@ Gem::Specification.new do |s| ## List your runtime dependencies here. Runtime dependencies are those ## that are needed for an end user to actually USE your code. s.add_dependency('syslog_protocol') + s.add_dependency('activesupport', '>= 3.2.14', '< 5.1') s.add_development_dependency('rake') s.add_development_dependency('test-unit') diff --git a/test/test_remote_syslog_logger.rb b/test/test_remote_syslog_logger.rb index c9ddb7f..060f1f7 100644 --- a/test/test_remote_syslog_logger.rb +++ b/test/test_remote_syslog_logger.rb @@ -1,4 +1,7 @@ +# encoding: utf-8 + require File.expand_path('../helper', __FILE__) +require File.expand_path('../../lib/remote_syslog_logger/limit_bytesize', __FILE__) class TestRemoteSyslogLogger < MiniTest::Test def setup @@ -49,7 +52,8 @@ def test_logger_long_default_tag TEST_HOSTNAME = 'bar' TEST_FACILITY = 'user' TEST_SEVERITY = 'notice' - TEST_MESSAGE = "abcdefg" * 512 + TEST_MESSAGE = "abcdefgâś“" * 512 + TEST_MESSAGE_ASCII8 = "abcdefg".force_encoding('ASCII') def test_logger_long_message _test_msg_splitting_with( @@ -84,6 +88,28 @@ def test_logger_long_message_custom_continuation continuation_prefix: 'frobnicate') end + def test_logger_ascii8_message + _test_msg_splitting_with( + tag: TEST_TAG, + hostname: TEST_HOSTNAME, + severity: TEST_SEVERITY, + facility: TEST_FACILITY, + message: TEST_MESSAGE_ASCII8, + max_packet_size: nil, + continuation_prefix: nil) + end + + def test_logger_empty_message + _test_msg_splitting_with( + tag: TEST_TAG, + hostname: TEST_HOSTNAME, + severity: TEST_SEVERITY, + facility: TEST_FACILITY, + message: '', + max_packet_size: nil, + continuation_prefix: nil) + end + private class MessageOnlyFormatter < ::Logger::Formatter @@ -114,14 +140,21 @@ def _test_msg_splitting_with(options) test_packet.content = '' max_content_size = packet_size - test_packet.assemble.size - offset = 0 line_prefix = '' - while offset < options[:message].size - chunk_size = max_content_size - line_prefix.size - message, addr = *@socket.recvfrom(packet_size * 2) - assert_match Regexp.new(': ' + line_prefix + Regexp.escape(options[:message][offset...offset + chunk_size]) + '$'), message - offset += chunk_size + remaining_message = options[:message] + reassembled_message = '' + until remaining_message.empty? + chunk_size = max_content_size - line_prefix.bytesize + chunk = limit_bytesize(remaining_message, chunk_size) + message, = *@socket.recvfrom(packet_size * 2) + message.force_encoding('UTF-8') + match = Regexp.new( + ': ' + line_prefix + '(' + Regexp.escape(chunk) + ')$').match(message) + assert !match.nil? + reassembled_message += match[1] + remaining_message = remaining_message[chunk.size..-1] line_prefix = continuation_prefix end + assert_equal(reassembled_message, options[:message]) end end From 9067b4f07058c9330b8a7c07515000f73a051bb8 Mon Sep 17 00:00:00 2001 From: Julian Scheid Date: Fri, 22 Feb 2019 21:28:07 +1300 Subject: [PATCH 5/5] Remove activesupport max version Tested to work with activesupport 5.2.2. --- remote_syslog_logger.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remote_syslog_logger.gemspec b/remote_syslog_logger.gemspec index fecef05..69b1f63 100644 --- a/remote_syslog_logger.gemspec +++ b/remote_syslog_logger.gemspec @@ -49,7 +49,7 @@ Gem::Specification.new do |s| ## List your runtime dependencies here. Runtime dependencies are those ## that are needed for an end user to actually USE your code. s.add_dependency('syslog_protocol') - s.add_dependency('activesupport', '>= 3.2.14', '< 5.1') + s.add_dependency('activesupport', '>= 3.2.14') s.add_development_dependency('rake') s.add_development_dependency('test-unit')