diff --git a/docs/created.rid b/docs/created.rid index 96ed13c..a1c56fb 100644 --- a/docs/created.rid +++ b/docs/created.rid @@ -1,16 +1,16 @@ -Wed, 06 Jul 2022 19:57:30 +0300 -lib/zstds.rb Wed, 06 Nov 2019 13:34:29 +0300 -lib/zstds/dictionary.rb Mon, 04 Jul 2022 11:54:49 +0300 -lib/zstds/error.rb Fri, 01 Jul 2022 17:12:27 +0300 -lib/zstds/file.rb Mon, 04 Jul 2022 11:55:58 +0300 -lib/zstds/option.rb Mon, 04 Jul 2022 12:53:26 +0300 -lib/zstds/stream/raw/compressor.rb Mon, 04 Jul 2022 11:39:13 +0300 -lib/zstds/stream/raw/decompressor.rb Fri, 01 Jul 2022 18:55:45 +0300 -lib/zstds/stream/reader.rb Fri, 01 Jul 2022 18:56:23 +0300 -lib/zstds/stream/writer.rb Fri, 01 Jul 2022 18:56:40 +0300 -lib/zstds/string.rb Mon, 04 Jul 2022 12:54:38 +0300 -lib/zstds/validation.rb Mon, 04 Jul 2022 12:55:11 +0300 -lib/zstds/version.rb Mon, 18 Apr 2022 17:57:48 +0300 -AUTHORS Mon, 18 Apr 2022 17:57:48 +0300 -LICENSE Tue, 29 Oct 2019 15:50:40 +0300 -README.md Fri, 01 Jul 2022 16:36:27 +0300 +Wed, 06 Jul 2022 23:32:21 +0300 +lib/zstds.rb Tue, 08 Sep 2020 12:25:29 +0300 +lib/zstds/dictionary.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/error.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/file.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/option.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/stream/raw/compressor.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/stream/raw/decompressor.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/stream/reader.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/stream/writer.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/string.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/validation.rb Wed, 06 Jul 2022 23:08:29 +0300 +lib/zstds/version.rb Thu, 31 Mar 2022 01:21:08 +0300 +AUTHORS Thu, 31 Mar 2022 01:21:08 +0300 +LICENSE Tue, 08 Sep 2020 12:25:29 +0300 +README.md Wed, 06 Jul 2022 23:08:29 +0300 diff --git a/docs/js/search_index.js.gz b/docs/js/search_index.js.gz index 171df6f..543d015 100644 Binary files a/docs/js/search_index.js.gz and b/docs/js/search_index.js.gz differ diff --git a/test/stream/reader.test.rb b/test/stream/reader.test.rb index 46421f5..7b2eca5 100644 --- a/test/stream/reader.test.rb +++ b/test/stream/reader.test.rb @@ -1,66 +1,24 @@ # Ruby bindings for zstd library. # Copyright (c) 2019 AUTHORS, MIT License. -require "adsp/test/stream/abstract" -require "set" -require "socket" +require "adsp/test/stream/reader" require "zstds/stream/reader" require "zstds/string" require "stringio" -require_relative "../common" require_relative "../minitest" require_relative "../option" -require_relative "../validation" module ZSTDS module Test module Stream - class Reader < ADSP::Test::Stream::Abstract + class Reader < ADSP::Test::Stream::Reader Target = ZSTDS::Stream::Reader + Option = Test::Option String = ZSTDS::String - ARCHIVE_PATH = Common::ARCHIVE_PATH - ENCODINGS = Common::ENCODINGS - TRANSCODE_OPTIONS = Common::TRANSCODE_OPTIONS - TEXTS = Common::TEXTS - LARGE_TEXTS = Common::LARGE_TEXTS - PORTION_LENGTHS = Common::PORTION_LENGTHS - LARGE_PORTION_LENGTHS = Common::LARGE_PORTION_LENGTHS - - BUFFER_LENGTH_NAMES = %i[source_buffer_length destination_buffer_length].freeze - BUFFER_LENGTH_MAPPING = { - :source_buffer_length => :destination_buffer_length, - :destination_buffer_length => :source_buffer_length - } - .freeze - - def test_invalid_initialize - get_invalid_decompressor_options do |invalid_options| - assert_raises ValidateError do - target.new ::StringIO.new, invalid_options - end - end - - super - end - - # -- synchronous -- - def test_invalid_read - instance = target.new ::StringIO.new - - (Validation::INVALID_NOT_NEGATIVE_INTEGERS - [nil]).each do |invalid_integer| - assert_raises ValidateError do - instance.read invalid_integer - end - end - - (Validation::INVALID_STRINGS - [nil]).each do |invalid_string| - assert_raises ValidateError do - instance.read nil, invalid_string - end - end + super corrupted_compressed_text = String.compress("1111").reverse instance = target.new ::StringIO.new(corrupted_compressed_text) @@ -68,309 +26,10 @@ def test_invalid_read assert_raises DecompressorCorruptedSourceError do instance.read end - - assert_raises ValidateError do - instance = target.new Validation::StringIOWithoutRead.new - instance.read - end - - assert_raises ValidateError do - instance = target.new Validation::StringIOWithoutEOF.new - instance.read - end - end - - def test_read - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - archive = get_archive text, compressor_options - prev_result = "".b - - Option::BOOLS.each do |with_buffer| - PORTION_LENGTHS.each do |portion_length| - get_compatible_decompressor_options compressor_options do |decompressor_options| - instance = target.new ::StringIO.new(archive), decompressor_options - decompressed_text = "".b - - begin - result = instance.read 0 - assert_equal "", result - - loop do - prev_eof = instance.eof? - - result = - if with_buffer - instance.read portion_length, prev_result - else - instance.read portion_length - end - - if result.nil? - assert_predicate instance, :eof? - break - end - - refute prev_eof unless archive.bytesize.zero? - - assert_equal prev_result, result if with_buffer - decompressed_text << result - end - - assert_equal instance.pos, decompressed_text.bytesize - assert_equal instance.pos, instance.tell - ensure - refute_predicate instance, :closed? - instance.close - assert_predicate instance, :closed? - end - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - end - - get_compatible_decompressor_options compressor_options do |decompressor_options| - instance = target.new ::StringIO.new(archive), decompressor_options - decompressed_text = nil - - begin - prev_eof = instance.eof? - - if with_buffer - decompressed_text = instance.read nil, prev_result - assert_equal prev_result, decompressed_text - else - decompressed_text = instance.read - end - - assert_predicate instance, :eof? - refute prev_eof unless archive.bytesize.zero? - - assert_equal instance.pos, decompressed_text.bytesize - assert_equal instance.pos, instance.tell - ensure - refute_predicate instance, :closed? - instance.close - assert_predicate instance, :closed? - end - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - end - end - end - end - - def test_read_with_large_texts - options_generator = OCG.new( - :text => LARGE_TEXTS, - :with_buffer => Option::BOOLS - ) - - Common.parallel_options options_generator do |options| - text = options[:text] - with_buffer = options[:with_buffer] - - archive = get_archive text - prev_result = "".b - - LARGE_PORTION_LENGTHS.each do |portion_length| - instance = target.new ::StringIO.new(archive) - decompressed_text = "".b - - begin - loop do - result = - if with_buffer - instance.read portion_length, prev_result - else - instance.read portion_length - end - - break if result.nil? - - assert_equal prev_result, result if with_buffer - decompressed_text << result - end - ensure - instance.close - end - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - - instance = target.new ::StringIO.new(archive) - decompressed_text = nil - - begin - if with_buffer - decompressed_text = instance.read nil, prev_result - assert_equal prev_result, decompressed_text - else - decompressed_text = instance.read - end - ensure - instance.close - end - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end end - def test_encoding - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - external_encoding = text.encoding - archive = get_archive text, compressor_options - - PORTION_LENGTHS.each do |portion_length| - get_compatible_decompressor_options compressor_options do |decompressor_options| - instance = target.new ::StringIO.new(archive), decompressor_options - decompressed_text = "".b - - begin - result = instance.read 0 - assert_equal Encoding::BINARY, result.encoding - - loop do - result = instance.read portion_length - break if result.nil? - - assert_equal Encoding::BINARY, result.encoding - decompressed_text << result - end - ensure - instance.close - end - - decompressed_text.force_encoding external_encoding - assert_equal text, decompressed_text - end - end - - # We don't need to transcode between same encodings. - (ENCODINGS - [external_encoding]).each do |internal_encoding| - target_text = text.encode internal_encoding, **TRANSCODE_OPTIONS - - get_compatible_decompressor_options compressor_options do |decompressor_options| - instance = target.new( - ::StringIO.new(archive), - decompressor_options, - :external_encoding => external_encoding, - :internal_encoding => internal_encoding, - :transcode_options => TRANSCODE_OPTIONS - ) - - assert_equal external_encoding, instance.external_encoding - assert_equal internal_encoding, instance.internal_encoding - assert_equal TRANSCODE_OPTIONS, instance.transcode_options - - decompressed_text = nil - - begin - instance.set_encoding external_encoding, internal_encoding, TRANSCODE_OPTIONS - assert_equal external_encoding, instance.external_encoding - assert_equal internal_encoding, instance.internal_encoding - assert_equal TRANSCODE_OPTIONS, instance.transcode_options - - decompressed_text = instance.read - assert_equal internal_encoding, decompressed_text.encoding - ensure - instance.close - end - - assert_equal target_text, decompressed_text - assert_predicate target_text, :valid_encoding? - end - end - end - end - end - - def test_rewind - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - get_compatible_decompressor_options compressor_options do |decompressor_options| - decompressed_text = nil - - ::File.open archive_path, "rb" do |file| - instance = target.new file, decompressor_options - - begin - result_1 = instance.read - - assert_equal 0, instance.rewind - assert_equal 0, instance.pos - assert_equal instance.pos, instance.tell - - result_2 = instance.read - assert_equal result_1, result_2 - - decompressed_text = result_1 - ensure - instance.close - end - end - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - end - end - end - - def test_invalid_eof - assert_raises ValidateError do - instance = target.new Validation::StringIOWithoutEOF.new - instance.eof? - end - end - - def test_eof - compressed_text = String.compress "ab" - instance = target.new ::StringIO.new(compressed_text) - - refute_predicate instance, :eof? - - byte = instance.read 1 - refute_predicate instance, :eof? - assert_equal "a", byte - - byte = instance.read 1 - assert_predicate instance, :eof? - assert_equal "b", byte - end - - # -- asynchronous -- - def test_invalid_readpartial_and_read_nonblock - instance = target.new ::StringIO.new - - Validation::INVALID_NOT_NEGATIVE_INTEGERS.each do |invalid_integer| - assert_raises ValidateError do - instance.readpartial invalid_integer - end - assert_raises ValidateError do - instance.read_nonblock invalid_integer - end - end - - (Validation::INVALID_STRINGS - [nil]).each do |invalid_string| - assert_raises ValidateError do - instance.readpartial 0, invalid_string - end - assert_raises ValidateError do - instance.read_nonblock 0, invalid_string - end - end + super corrupted_compressed_text = String.compress("1111").reverse @@ -385,276 +44,6 @@ def test_invalid_readpartial_and_read_nonblock assert_raises DecompressorCorruptedSourceError do instance.read_nonblock 1 end - - assert_raises ValidateError do - instance = target.new Validation::StringIOWithoutReadpartial.new - instance.readpartial 1 - end - - assert_raises ValidateError do - instance = target.new Validation::StringIOWithoutReadNonblock.new - instance.read_nonblock 1 - end - end - - def test_readpartial - IO.pipe do |read_io, write_io| - instance = target.new read_io - write_io.close - - assert_raises ::EOFError do - instance.readpartial 1 - end - end - - nonblock_server do |server| - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - Option::BOOLS.each do |with_buffer| - nonblock_test server, text, portion_length, compressor_options do |instance| - prev_result = "".b - decompressed_text = "".b - - loop do - if with_buffer - result = instance.readpartial portion_length, prev_result - assert_equal prev_result, result - else - result = instance.readpartial portion_length - end - - decompressed_text << result - rescue ::EOFError - break - end - - decompressed_text - end - end - end - end - end - end - end - - def test_read_nonblock - IO.pipe do |read_io, write_io| - instance = target.new read_io - - assert_raises ::IO::WaitReadable do - instance.read_nonblock 1 - end - - write_io.close - - assert_raises ::EOFError do - instance.read_nonblock 1 - end - end - - nonblock_server do |server| - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - nonblock_test server, text, portion_length, compressor_options do |instance, socket| - decompressed_text = "".b - - loop do - begin - decompressed_text << instance.read_nonblock(portion_length) - rescue ::IO::WaitReadable - socket.wait_readable - retry - rescue ::EOFError - break - end - - begin - decompressed_text << instance.readpartial(portion_length) - rescue ::EOFError - break - end - - result = instance.read portion_length - break if result.nil? - - decompressed_text << result - end - - decompressed_text - end - end - end - end - end - end - - def test_read_nonblock_with_large_texts - nonblock_server do |server| - Common.parallel LARGE_TEXTS do |text| - LARGE_PORTION_LENGTHS.each do |portion_length| - nonblock_test server, text, portion_length do |instance, socket| - decompressed_text = "".b - - loop do - begin - decompressed_text << instance.read_nonblock(portion_length) - rescue ::IO::WaitReadable - socket.wait_readable - retry - rescue ::EOFError - break - end - - begin - decompressed_text << instance.readpartial(portion_length) - rescue ::EOFError - break - end - - result = instance.read portion_length - break if result.nil? - - decompressed_text << result - end - - decompressed_text - end - end - end - end - end - - # -- nonblock test -- - - protected def nonblock_server - # Server need just to redirect content for client. - - ::TCPServer.open 0 do |server| - # Server loop will be processed in separate (parent) thread. - # Child threads will be collected for later usage. - child_lock = ::Mutex.new - child_threads = ::Set.new - - parent_thread = ::Thread.new do - loop do - child_thread = ::Thread.start server.accept do |socket| - result = "".b - - # Reading head. - result_size, portion_length = socket.read(16).unpack "QQ" - next if result_size.zero? - - # Reading result. - loop do - begin - result << socket.read_nonblock(portion_length) - rescue ::IO::WaitReadable - socket.wait_readable - retry - end - - break if result.bytesize == result_size - end - - # Writing result. - loop do - begin - bytes_written = socket.write_nonblock result - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - result = result.byteslice bytes_written, result.bytesize - bytes_written - result_size -= bytes_written - - break if result_size.zero? - end - - ensure - socket.close - - # Removing current child thread. - child_lock.synchronize { child_threads.delete ::Thread.current } - end - - # Adding new child thread. - child_lock.synchronize { child_threads.add child_thread } - end - end - - # Processing client. - begin - yield server - ensure - # We need to kill parent thread when client has finished. - # So server won't be able to create new child threads. - # Than we can join all remaining child threads. - parent_thread.kill.join - child_threads.each(&:join) - end - end - end - - protected def nonblock_test(server, text, portion_length, compressor_options = {}, &_block) - port = server.addr[1] - compressed_text = String.compress text, compressor_options - - processor = proc do |decompressor_options| - decompressed_text = ::TCPSocket.open "localhost", port do |socket| - # Writing head. - head = [compressed_text.bytesize, portion_length].pack "QQ" - socket.write head - - # Writing compressed text. - socket.write compressed_text - - instance = target.new socket, decompressor_options - - begin - yield instance, socket - ensure - instance.close - end - end - - # Testing decompressed text. - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - - if compressor_options.empty? - processor.call({}) - else - get_compatible_decompressor_options compressor_options do |decompressor_options| - processor.call decompressor_options - end - end - end - - # ----- - - protected def get_archive(text, compressor_options = {}) - String.compress text, compressor_options - end - - protected def write_archive(archive_path, text, compressor_options = {}) - compressed_text = String.compress text, compressor_options - ::File.write archive_path, compressed_text, :mode => "wb" - end - - def get_invalid_decompressor_options(&block) - Option.get_invalid_decompressor_options BUFFER_LENGTH_NAMES, &block - end - - def parallel_compressor_options(&block) - Common.parallel_options Option.get_compressor_options_generator(BUFFER_LENGTH_NAMES), &block - end - - def get_compatible_decompressor_options(compressor_options, &block) - Option.get_compatible_decompressor_options compressor_options, BUFFER_LENGTH_MAPPING, &block end end diff --git a/test/stream/reader_helpers.test.rb b/test/stream/reader_helpers.test.rb index bd43c83..fcccdba 100644 --- a/test/stream/reader_helpers.test.rb +++ b/test/stream/reader_helpers.test.rb @@ -1,418 +1,20 @@ # Ruby bindings for zstd library. # Copyright (c) 2019 AUTHORS, MIT License. -require "English" -require "stringio" +require "adsp/test/stream/reader_helpers" require "zstds/stream/reader" require "zstds/string" -require_relative "../common" require_relative "../minitest" require_relative "../option" -require_relative "../validation" module ZSTDS module Test module Stream - class ReaderHelpers < Minitest::Test + class ReaderHelpers < ADSP::Test::Stream::ReaderHelpers Target = ZSTDS::Stream::Reader + Option = Test::Option String = ZSTDS::String - - ARCHIVE_PATH = Common::ARCHIVE_PATH - ENCODINGS = Common::ENCODINGS - TRANSCODE_OPTIONS = Common::TRANSCODE_OPTIONS - TEXTS = Common::TEXTS - LARGE_TEXTS = Common::LARGE_TEXTS - - BUFFER_LENGTH_NAMES = %i[source_buffer_length destination_buffer_length].freeze - BUFFER_LENGTH_MAPPING = { - :source_buffer_length => :destination_buffer_length, - :destination_buffer_length => :source_buffer_length - } - .freeze - - LIMITS = [nil, 1].freeze - - def test_invalid_ungetbyte - instance = target.new ::StringIO.new - - Validation::INVALID_STRINGS.each do |invalid_string| - assert_raises ValidateError do - instance.ungetbyte invalid_string - end - end - end - - def test_byte - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - get_compatible_decompressor_options compressor_options do |decompressor_options| - Target.open archive_path, decompressor_options do |instance| - # getbyte - - byte = instance.getbyte - instance.ungetbyte byte unless byte.nil? - - # readbyte - - begin - byte = instance.readbyte - instance.ungetc byte - rescue ::EOFError - # ok - end - - # each_byte - - decompressed_text = "".b - instance.each_byte { |current_byte| decompressed_text << current_byte } - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - end - end - end - end - - # -- char -- - - def test_invalid_ungetc - instance = target.new ::StringIO.new - - Validation::INVALID_STRINGS.each do |invalid_string| - assert_raises ValidateError do - instance.ungetc invalid_string - end - end - end - - def test_char - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - get_compatible_decompressor_options compressor_options do |decompressor_options| - Target.open archive_path, decompressor_options do |instance| - # getc - - char = instance.getc - instance.ungetc char unless char.nil? - - # readchar - - begin - char = instance.readchar - instance.ungetc char - rescue ::EOFError - # ok - end - - # each_char - - decompressed_text = "".b - instance.each_char { |current_char| decompressed_text << current_char } - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - end - end - end - end - - def test_char_encoding - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - external_encoding = text.encoding - - (ENCODINGS - [external_encoding]).each do |internal_encoding| - target_text = text.encode internal_encoding, **TRANSCODE_OPTIONS - - get_compatible_decompressor_options compressor_options do |decompressor_options| - Target.open archive_path, decompressor_options do |instance| - instance.set_encoding external_encoding, internal_encoding, TRANSCODE_OPTIONS - - # getc - - char = instance.getc - - unless char.nil? - assert_equal internal_encoding, char.encoding - instance.ungetc char - end - - # readchar - - begin - char = instance.readchar - assert_equal internal_encoding, char.encoding - - instance.ungetc char - rescue ::EOFError - # ok - end - - # each_char - - decompressed_text = ::String.new :encoding => internal_encoding - - instance.each_char do |current_char| - assert_equal internal_encoding, current_char.encoding - decompressed_text << current_char - end - - assert_equal target_text, decompressed_text - end - end - end - end - end - end - - # -- lines -- - - def test_invalid_gets - instance = target.new ::StringIO.new - - (Validation::INVALID_STRINGS - [nil, 1, 1.1]).each do |invalid_string| - assert_raises ValidateError do - instance.gets invalid_string - end - end - - (Validation::INVALID_POSITIVE_INTEGERS - [nil]).each do |invalid_integer| - assert_raises ValidateError do - instance.gets nil, invalid_integer - end - end - end - - def test_invalid_ungetline - instance = target.new ::StringIO.new - - Validation::INVALID_STRINGS.each do |invalid_string| - assert_raises ValidateError do - instance.ungetline invalid_string - end - end - end - - def test_lines - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - separator = - if text.empty? - nil - else - text[0] - end - - get_compatible_decompressor_options compressor_options do |decompressor_options| - Target.open archive_path, decompressor_options do |instance| - # lineno - - assert_equal 0, instance.lineno - - instance.lineno = 1 - assert_equal 1, instance.lineno - - instance.lineno = 0 - assert_equal 0, instance.lineno - - # gets - - LIMITS.each do |limit| - line = instance.gets limit - next if line.nil? - - assert_equal 1, instance.lineno - - instance.ungetline line - assert_equal 0, instance.lineno - - # Same test with separator. - - line = instance.gets separator, limit - next if line.nil? - - assert_equal 1, instance.lineno - - instance.ungetline line - assert_equal 0, instance.lineno - end - - # readline - - begin - line = instance.readline - assert_equal 1, instance.lineno - - instance.ungetline line - assert_equal 0, instance.lineno - rescue ::EOFError - # ok - end - - # readlines - - lines = instance.readlines - lines.each { |current_line| instance.ungetline current_line } - - decompressed_text = lines.join - decompressed_text.force_encoding text.encoding - - assert_equal text, decompressed_text - - # each_line - - decompressed_text = "".b - instance.each_line { |current_line| decompressed_text << current_line } - - decompressed_text.force_encoding text.encoding - assert_equal text, decompressed_text - end - end - end - end - end - - def test_lines_encoding - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - external_encoding = text.encoding - - (ENCODINGS - [external_encoding]).each do |internal_encoding| - target_text = text.encode internal_encoding, **TRANSCODE_OPTIONS - - separator = - if text.empty? - nil - else - text[0] - end - - get_compatible_decompressor_options compressor_options do |decompressor_options| - Target.open archive_path, decompressor_options do |instance| - instance.set_encoding external_encoding, internal_encoding, TRANSCODE_OPTIONS - - # gets - - line = instance.gets separator - - unless line.nil? - assert_equal internal_encoding, line.encoding - instance.ungetline line - end - - # readline - - begin - line = instance.readline - assert_equal internal_encoding, line.encoding - - instance.ungetline line - rescue ::EOFError - # ok - end - - # each_line - - decompressed_text = ::String.new :encoding => internal_encoding - - instance.each_line do |current_line| - assert_equal internal_encoding, current_line.encoding - decompressed_text << current_line - end - - assert_equal target_text, decompressed_text - end - end - end - end - end - end - - # -- etc -- - - def test_invalid_open - Validation::INVALID_STRINGS.each do |invalid_string| - assert_raises ValidateError do - Target.open(invalid_string) {} # no-op - end - end - - # Proc is required. - assert_raises ValidateError do - Target.open ARCHIVE_PATH - end - end - - def test_open - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - write_archive archive_path, text, compressor_options - - get_compatible_decompressor_options compressor_options do |decompressor_options| - decompressed_text = Target.open archive_path, decompressor_options, &:read - decompressed_text.force_encoding text.encoding - - assert_equal text, decompressed_text - end - end - end - end - - def test_open_with_large_texts - Common.parallel LARGE_TEXTS do |text, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - write_archive archive_path, text - - decompressed_text = Target.open archive_path, &:read - decompressed_text.force_encoding text.encoding - - assert_equal text, decompressed_text - end - end - - # ----- - - protected def write_archive(archive_path, text, compressor_options = {}) - compressed_text = String.compress text, compressor_options - ::File.write archive_path, compressed_text, :mode => "wb" - end - - def parallel_compressor_options(&block) - Common.parallel_options Option.get_compressor_options_generator(BUFFER_LENGTH_NAMES), &block - end - - def get_compatible_decompressor_options(compressor_options, &block) - Option.get_compatible_decompressor_options compressor_options, BUFFER_LENGTH_MAPPING, &block - end - - protected def target - self.class::Target - end end Minitest << ReaderHelpers diff --git a/test/stream/writer.test.rb b/test/stream/writer.test.rb index ce42997..6005d09 100644 --- a/test/stream/writer.test.rb +++ b/test/stream/writer.test.rb @@ -1,613 +1,20 @@ # Ruby bindings for zstd library. # Copyright (c) 2019 AUTHORS, MIT License. -require "adsp/test/stream/abstract" -require "set" -require "socket" -require "stringio" +require "adsp/test/stream/writer" require "zstds/stream/writer" require "zstds/string" -require_relative "../common" require_relative "../minitest" require_relative "../option" module ZSTDS module Test module Stream - class Writer < ADSP::Test::Stream::Abstract + class Writer < ADSP::Test::Stream::Writer Target = ZSTDS::Stream::Writer + Option = Test::Option String = ZSTDS::String - - ARCHIVE_PATH = Common::ARCHIVE_PATH - ENCODINGS = Common::ENCODINGS - TRANSCODE_OPTIONS = Common::TRANSCODE_OPTIONS - TEXTS = Common::TEXTS - LARGE_TEXTS = Common::LARGE_TEXTS - PORTION_LENGTHS = Common::PORTION_LENGTHS - LARGE_PORTION_LENGTHS = Common::LARGE_PORTION_LENGTHS - - BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze - BUFFER_LENGTH_MAPPING = { :destination_buffer_length => :destination_buffer_length }.freeze - FINISH_MODES = OCG.new( - :flush_nonblock => Option::BOOLS, - :close_nonblock => Option::BOOLS - ) - .freeze - - NONBLOCK_SERVER_MODES = { - :request => 0, - :response => 1 - } - .freeze - - NONBLOCK_SERVER_TIMEOUT = 0.1 - - def initialize(*args) - super(*args) - - @nonblock_client_lock = ::Mutex.new - @nonblock_client_id = 0 - end - - def test_invalid_initialize - get_invalid_compressor_options do |invalid_options| - assert_raises ValidateError do - target.new ::StringIO.new, invalid_options - end - end - - (Validation::INVALID_NOT_NEGATIVE_INTEGERS - [nil]).each do |invalid_integer| - assert_raises ValidateError do - target.new ::StringIO.new, :pledged_size => invalid_integer - end - end - - super - end - - # -- synchronous -- - - def test_invalid_write - instance = target.new Validation::StringIOWithoutWrite.new - - assert_raises ValidateError do - instance.write "" - end - - assert_raises ValidateError do - instance.flush - end - - assert_raises ValidateError do - instance.rewind - end - - assert_raises ValidateError do - instance.close - end - end - - def test_write - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - sources = get_sources text, portion_length - io = ::StringIO.new - instance = target.new io, compressor_options.merge(:pledged_size => text.bytesize) - - begin - sources.each_slice 2 do |current_sources| - instance.write(*current_sources) - instance.flush - end - - assert_equal instance.pos, text.bytesize - assert_equal instance.pos, instance.tell - ensure - refute_predicate instance, :closed? - instance.close - assert_predicate instance, :closed? - end - - compressed_text = io.string - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - end - - def test_write_with_large_texts - options_generator = OCG.new( - :text => LARGE_TEXTS, - :portion_length => LARGE_PORTION_LENGTHS - ) - - Common.parallel_options options_generator do |options| - text = options[:text] - portion_length = options[:portion_length] - - sources = get_sources text, portion_length - io = ::StringIO.new - instance = target.new io - - begin - sources.each_slice 2 do |current_sources| - instance.write(*current_sources) - instance.flush - end - ensure - instance.close - end - - compressed_text = io.string - check_text text, compressed_text - end - end - - def test_encoding - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - # We don't need to transcode between same encodings. - (ENCODINGS - [text.encoding]).each do |external_encoding| - target_text = text.encode external_encoding, **TRANSCODE_OPTIONS - io = ::StringIO.new - - instance = target.new( - io, - compressor_options.merge(:pledged_size => target_text.bytesize), - :external_encoding => external_encoding, - :transcode_options => TRANSCODE_OPTIONS - ) - - assert_equal external_encoding, instance.external_encoding - assert_equal TRANSCODE_OPTIONS, instance.transcode_options - - begin - instance.set_encoding external_encoding, nil, TRANSCODE_OPTIONS - assert_equal external_encoding, instance.external_encoding - assert_equal TRANSCODE_OPTIONS, instance.transcode_options - - instance.write text - ensure - instance.close - end - - compressed_text = io.string - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text target_text, compressed_text, decompressor_options - assert_predicate target_text, :valid_encoding? - end - end - end - end - end - - def test_rewind - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - compressed_texts = [] - - ::File.open archive_path, "wb" do |file| - instance = target.new file, compressor_options - - begin - TEXTS.each do |text| - instance.write text - instance.flush - - assert_equal instance.pos, text.bytesize - assert_equal instance.pos, instance.tell - - assert_equal 0, instance.rewind - - compressed_texts << ::File.read(archive_path, :mode => "rb") - - assert_equal 0, instance.pos - assert_equal instance.pos, instance.tell - - file.truncate 0 - end - ensure - instance.close - end - end - - TEXTS.each.with_index do |text, index| - compressed_text = compressed_texts[index] - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - - # -- asynchronous -- - - def test_invalid_write_nonblock - instance = target.new Validation::StringIOWithoutWriteNonblock.new - - assert_raises ValidateError do - instance.write_nonblock "" - end - - assert_raises ValidateError do - instance.flush_nonblock - end - - assert_raises ValidateError do - instance.close_nonblock - end - end - - def test_write_nonblock - nonblock_server do |server| - parallel_compressor_options do |compressor_options| - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - sources = get_sources text, portion_length - - FINISH_MODES.each do |finish_mode| - nonblock_test server, text, portion_length, compressor_options do |instance, socket| - # write - - sources.each.with_index do |source, index| - if index.even? - loop do - begin - bytes_written = instance.write_nonblock source - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - source = source.byteslice bytes_written, source.bytesize - bytes_written - break if source.bytesize.zero? - end - else - instance.write source - end - end - - # flush - - if finish_mode[:flush_nonblock] - loop do - begin - is_flushed = instance.flush_nonblock - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - break if is_flushed - end - else - instance.flush - end - - assert_equal instance.pos, text.bytesize - assert_equal instance.pos, instance.tell - - ensure - # close - - refute_predicate instance, :closed? - - if finish_mode[:close_nonblock] - loop do - begin - is_closed = instance.close_nonblock - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - break if is_closed - end - else - instance.close - end - - assert_predicate instance, :closed? - end - end - end - end - end - end - end - - def test_write_nonblock_with_large_texts - nonblock_server do |server| - Common.parallel LARGE_TEXTS do |text| - LARGE_PORTION_LENGTHS.each do |portion_length| - sources = get_sources text, portion_length - - FINISH_MODES.each do |finish_mode| - nonblock_test server, text, portion_length do |instance, socket| - # write - - sources.each.with_index do |source, index| - if index.even? - loop do - begin - bytes_written = instance.write_nonblock source - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - source = source.byteslice bytes_written, source.bytesize - bytes_written - break if source.bytesize.zero? - end - else - instance.write source - end - end - - # flush - - if finish_mode[:flush_nonblock] - loop do - begin - is_flushed = instance.flush_nonblock - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - break if is_flushed - end - else - instance.flush - end - - ensure - # close - - if finish_mode[:close_nonblock] - loop do - begin - is_closed = instance.close_nonblock - rescue ::IO::WaitWritable - socket.wait_writable - retry - end - - break if is_closed - end - else - instance.close - end - end - end - end - end - end - end - - def test_invalid_rewind_nonblock - instance = target.new Validation::StringIOWithoutWriteNonblock.new - - assert_raises ValidateError do - instance.rewind_nonblock - end - end - - def test_rewind_nonblock - return unless Common.file_can_be_used_nonblock? - - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - compressed_texts = [] - - ::File.open archive_path, "wb" do |file| - instance = target.new file, compressor_options - - begin - TEXTS.each do |text| - instance.write text - instance.flush - - assert_equal instance.pos, text.bytesize - assert_equal instance.pos, instance.tell - - loop do - begin - is_rewinded = instance.rewind_nonblock - rescue ::IO::WaitWritable - file.wait_writable - retry - end - - break if is_rewinded - end - - compressed_texts << ::File.read(archive_path, :mode => "rb") - - assert_equal 0, instance.pos - assert_equal instance.pos, instance.tell - - file.truncate 0 - end - ensure - instance.close - end - end - - TEXTS.each.with_index do |text, index| - compressed_text = compressed_texts[index] - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - - # -- nonblock test -- - - protected def nonblock_server - # We need to test close nonblock. - # This method writes remaining data and closes socket. - # Server is not able to send response immediately. - # Client has to reconnect to server once again. - - ::TCPServer.open 0 do |server| - # Server loop will be processed in separate (parent) thread. - # Child threads will be collected for later usage. - child_lock = ::Mutex.new - child_threads = ::Set.new - - # Server need to maintain mapping between client id and result. - results_lock = ::Mutex.new - results = {} - - parent_thread = ::Thread.new do - loop do - child_thread = ::Thread.start server.accept do |socket| - # Reading head. - client_id, portion_length, mode = socket.read(13).unpack "NQC" - - if mode == NONBLOCK_SERVER_MODES[:request] - # Reading result from client. - result = "".b - - loop do - result << socket.read_nonblock(portion_length) - rescue ::IO::WaitReadable - socket.wait_readable - rescue ::EOFError - break - end - - # Saving result for client. - results_lock.synchronize { results[client_id] = result } - - next - end - - loop do - # Waiting when result will be ready. - result = results_lock.synchronize { results[client_id] } - - unless result.nil? - # Sending result to client. - socket.write result - - break - end - - sleep NONBLOCK_SERVER_TIMEOUT - end - - # Removing result for client. - results_lock.synchronize { results.delete client_id } - - ensure - socket.close - - # Removing current child thread. - child_lock.synchronize { child_threads.delete ::Thread.current } - end - - # Adding new child thread. - child_lock.synchronize { child_threads.add child_thread } - end - end - - # Processing client. - begin - yield server - ensure - # We need to kill parent thread when client has finished. - # So server won't be able to create new child threads. - # Than we can join all remaining child threads. - parent_thread.kill.join - child_threads.each(&:join) - end - end - end - - protected def nonblock_test(server, text, portion_length, compressor_options = {}, &_block) - port = server.addr[1] - client_id = @nonblock_client_lock.synchronize { @nonblock_client_id += 1 } - - # Writing request. - ::TCPSocket.open "localhost", port do |socket| - # Writing head. - head = [client_id, portion_length, NONBLOCK_SERVER_MODES[:request]].pack "NQC" - socket.write head - - # Instance is going to write compressed text. - instance = target.new socket, compressor_options - - begin - yield instance, socket - ensure - instance.close - end - end - - # Reading response. - compressed_text = ::TCPSocket.open "localhost", port do |socket| - # Writing head. - head = [client_id, portion_length, NONBLOCK_SERVER_MODES[:response]].pack "NQC" - socket.write head - - # Reading compressed text. - socket.read - end - - # Testing compressed text. - if compressor_options.empty? - check_text text, compressed_text - else - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - - # ----- - - protected def get_sources(text, portion_length) - sources = text - .chars - .each_slice(portion_length) - .map(&:join) - - return ["".b] if sources.empty? - - sources - end - - protected def check_text(text, compressed_text, decompressor_options = {}) - decompressed_text = String.decompress compressed_text, decompressor_options - decompressed_text.force_encoding text.encoding - - assert_equal text, decompressed_text - end - - def get_invalid_compressor_options(&block) - Option.get_invalid_compressor_options BUFFER_LENGTH_NAMES, &block - end - - def parallel_compressor_options(&block) - Common.parallel_options Option.get_compressor_options_generator(BUFFER_LENGTH_NAMES), &block - end - - def get_compatible_decompressor_options(compressor_options, &block) - Option.get_compatible_decompressor_options compressor_options, BUFFER_LENGTH_MAPPING, &block - end end Minitest << Writer diff --git a/test/stream/writer_helpers.test.rb b/test/stream/writer_helpers.test.rb index 5f4b62a..c766b93 100644 --- a/test/stream/writer_helpers.test.rb +++ b/test/stream/writer_helpers.test.rb @@ -1,264 +1,20 @@ # Ruby bindings for zstd library. # Copyright (c) 2019 AUTHORS, MIT License. -require "English" -require "stringio" +require "adsp/test/stream/writer_helpers" require "zstds/stream/writer" require "zstds/string" -require_relative "../common" require_relative "../minitest" require_relative "../option" -require_relative "../validation" module ZSTDS module Test module Stream - class WriterHelpers < Minitest::Test + class WriterHelpers < ADSP::Test::Stream::WriterHelpers Target = ZSTDS::Stream::Writer + Option = Test::Option String = ZSTDS::String - - ARCHIVE_PATH = Common::ARCHIVE_PATH - TEXTS = Common::TEXTS - LARGE_TEXTS = Common::LARGE_TEXTS - PORTION_LENGTHS = Common::PORTION_LENGTHS - LARGE_PORTION_LENGTHS = Common::LARGE_PORTION_LENGTHS - - BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze - BUFFER_LENGTH_MAPPING = { :destination_buffer_length => :destination_buffer_length }.freeze - - def test_write - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - sources = get_sources text, portion_length - - Target.open archive_path, compressor_options do |instance| - sources.each { |current_source| instance << current_source } - end - - compressed_text = ::File.read archive_path, :mode => "rb" - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - end - - def test_print - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.reject(&:empty?).each do |text| - PORTION_LENGTHS.each do |portion_length| - sources = get_sources text, portion_length - field_separator = " ".encode text.encoding - record_separator = "\n".encode text.encoding - - target_text = "".encode text.encoding - sources.each { |source| target_text << (source + field_separator) } - target_text << record_separator - - Target.open archive_path, compressor_options do |instance| - keyword_args = { :field_separator => field_separator, :record_separator => record_separator } - instance.print(*sources, **keyword_args) - end - - compressed_text = ::File.read archive_path, :mode => "rb" - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text target_text, compressed_text, decompressor_options - end - end - end - end - end - - def test_printf - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - sources = get_sources text, portion_length - - Target.open archive_path, compressor_options do |instance| - sources.each { |source| instance.printf "%s", source } - end - - compressed_text = ::File.read archive_path, :mode => "rb" - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - end - - def test_invalid_putc - instance = target.new ::StringIO.new - - Validation::INVALID_CHARS.each do |invalid_char| - assert_raises ValidateError do - instance.putc invalid_char - end - end - end - - def test_putc - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - Target.open archive_path, compressor_options do |instance| - # Putc should process numbers and strings. - text.chars.each.with_index do |char, index| - if index.even? - instance.putc char.ord, :encoding => text.encoding - else - instance.putc char - end - end - end - - compressed_text = ::File.read archive_path, :mode => "rb" - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - - def test_puts - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - PORTION_LENGTHS.each do |portion_length| - newline = "\n".encode text.encoding - - sources = get_sources text, portion_length - sources = sources.map do |source| - source.delete_suffix! newline while source.end_with? newline - source - end - - target_text = "".encode text.encoding - sources.each { |source| target_text << (source + newline) } - - Target.open archive_path, compressor_options do |instance| - # Puts should ignore additional newlines and process arrays. - args = sources.map.with_index do |source, index| - if index.even? - source + newline - else - [source] - end - end - - instance.puts(*args) - end - - compressed_text = ::File.read archive_path, :mode => "rb" - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text target_text, compressed_text, decompressor_options - end - end - end - end - end - - def test_invalid_open - Validation::INVALID_STRINGS.each do |invalid_string| - assert_raises ValidateError do - Target.open(invalid_string) {} # no-op - end - end - - # Proc is required. - assert_raises ValidateError do - Target.open ARCHIVE_PATH - end - end - - def test_open - parallel_compressor_options do |compressor_options, worker_index| - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - TEXTS.each do |text| - Target.open(archive_path, compressor_options) { |instance| instance.write text } - - compressed_text = ::File.read archive_path, :mode => "rb" - - get_compatible_decompressor_options compressor_options do |decompressor_options| - check_text text, compressed_text, decompressor_options - end - end - end - end - - def test_open_with_large_texts - options_generator = OCG.new( - :text => LARGE_TEXTS, - :portion_length => LARGE_PORTION_LENGTHS - ) - - Common.parallel_options options_generator do |options, worker_index| - text = options[:text] - portion_length = options[:portion_length] - - archive_path = Common.get_path ARCHIVE_PATH, worker_index - - sources = get_sources text, portion_length - - Target.open archive_path do |instance| - sources.each { |source| instance.write source } - end - - compressed_text = ::File.read archive_path, :mode => "rb" - - check_text text, compressed_text, {} - end - end - - # ----- - - protected def get_sources(text, portion_length) - sources = text - .chars - .each_slice(portion_length) - .map(&:join) - - return [""] if sources.empty? - - sources - end - - protected def check_text(text, compressed_text, decompressor_options) - decompressed_text = String.decompress compressed_text, decompressor_options - decompressed_text.force_encoding text.encoding - - assert_equal text, decompressed_text - end - - def parallel_compressor_options(&block) - Common.parallel_options Option.get_compressor_options_generator(BUFFER_LENGTH_NAMES), &block - end - - def get_compatible_decompressor_options(compressor_options, &block) - Option.get_compatible_decompressor_options compressor_options, BUFFER_LENGTH_MAPPING, &block - end - - protected def target - self.class::Target - end end Minitest << WriterHelpers