diff --git a/Cargo.lock b/Cargo.lock index 23cba49..40849ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,7 +191,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -481,7 +481,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -657,10 +657,10 @@ dependencies = [ "oval", "positioned-io", "pretty-hex", + "test-log", "thiserror", "tracing", "tracing-subscriber", - "tracing-test", "winnow", "zstd", ] @@ -732,7 +732,7 @@ checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -775,9 +775,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -785,14 +785,24 @@ dependencies = [ ] [[package]] -name = "syn" -version = "2.0.48" +name = "test-log" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "6159ab4116165c99fc88cce31f99fa2c9dbe08d3691cb38da02fc3b45f357d2b" +dependencies = [ + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-macros" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba277e77219e9eea169e8508942db1bf5d8a41ff2db9b20aab5a5aadc9fa25d" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", ] [[package]] @@ -812,7 +822,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -861,7 +871,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -903,29 +913,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "tracing-test" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" -dependencies = [ - "lazy_static", - "tracing-core", - "tracing-subscriber", - "tracing-test-macro", -] - -[[package]] -name = "tracing-test-macro" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" -dependencies = [ - "lazy_static", - "quote", - "syn 1.0.109", -] - [[package]] name = "unicode-ident" version = "1.0.12" @@ -971,7 +958,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn", "wasm-bindgen-shared", ] @@ -993,7 +980,7 @@ checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index aa7ebd1..d2b8daa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,8 +54,8 @@ zstd = ["dep:zstd"] clap = { version = "4.4.18", features = ["derive"] } humansize = "2.1.3" indicatif = "0.17.7" -tracing-subscriber = "0.3.18" -tracing-test = "0.2.4" +test-log = { version = "0.2.14", default-features = false, features = ["tracing-subscriber", "trace"] } +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } [profile.release] debug = 1 diff --git a/src/reader/archive_reader.rs b/src/reader/archive_reader.rs index 1ffd53e..0108142 100644 --- a/src/reader/archive_reader.rs +++ b/src/reader/archive_reader.rs @@ -73,8 +73,7 @@ impl ArchiveReaderState { impl ArchiveReader { /// This should be > 65KiB, because the section at the end of the /// file that we check for end of central directory record is 65KiB. - /// 128 is the next power of two. - const DEFAULT_BUFFER_SIZE: usize = 128 * 1024; + const DEFAULT_BUFFER_SIZE: usize = 256 * 1024; /// Create a new archive reader with a specified file size. /// diff --git a/src/reader/sync/decoder.rs b/src/reader/sync/decoder.rs index f2f70cc..a487681 100644 --- a/src/reader/sync/decoder.rs +++ b/src/reader/sync/decoder.rs @@ -54,12 +54,12 @@ where /// Only allows reading a fixed number of bytes from a [oval::Buffer], /// allowing to move the inner reader out afterwards. -pub struct LimitedReader { +pub struct RawEntryReader { remaining: u64, inner: Buffer, } -impl LimitedReader { +impl RawEntryReader { pub fn new(inner: Buffer, remaining: u64) -> Self { Self { inner, remaining } } @@ -73,7 +73,24 @@ impl LimitedReader { } } -impl io::Read for LimitedReader { +impl io::BufRead for RawEntryReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + tracing::trace!( + remaining = self.remaining, + available_data = self.inner.available_data(), + available_space = self.inner.available_space(), + "fill_buf" + ); + Ok(self.inner.data()) + } + + fn consume(&mut self, amt: usize) { + tracing::trace!(amt, "consume"); + Buffer::consume(&mut self.inner, amt); + } +} + +impl io::Read for RawEntryReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { let len = cmp::min(buf.len() as u64, self.remaining) as usize; tracing::trace!(%len, buf_len = buf.len(), remaining = self.remaining, available_data = self.inner.available_data(), available_space = self.inner.available_space(), "computing len"); diff --git a/src/reader/sync/entry_reader/bzip2_dec.rs b/src/reader/sync/entry_reader/bzip2_dec.rs index c8c0bc8..238427f 100644 --- a/src/reader/sync/entry_reader/bzip2_dec.rs +++ b/src/reader/sync/entry_reader/bzip2_dec.rs @@ -2,7 +2,7 @@ use std::io::Read; use bzip2::read::BzDecoder; -use crate::reader::sync::{Decoder, LimitedReader}; +use crate::reader::sync::{Decoder, RawEntryReader}; impl Decoder for BzDecoder where @@ -17,6 +17,6 @@ where } } -pub(crate) fn mk_decoder(r: LimitedReader) -> impl Decoder { +pub(crate) fn mk_decoder(r: RawEntryReader) -> impl Decoder { BzDecoder::new(r) } diff --git a/src/reader/sync/entry_reader/deflate64_dec.rs b/src/reader/sync/entry_reader/deflate64_dec.rs index 43ac3cd..cb1fdd0 100644 --- a/src/reader/sync/entry_reader/deflate64_dec.rs +++ b/src/reader/sync/entry_reader/deflate64_dec.rs @@ -2,7 +2,7 @@ use std::io::{BufReader, Read}; use deflate64::Deflate64Decoder; -use crate::reader::sync::{Decoder, LimitedReader}; +use crate::reader::sync::{Decoder, RawEntryReader}; impl Decoder for Deflate64Decoder> where @@ -17,6 +17,6 @@ where } } -pub(crate) fn mk_decoder(r: LimitedReader) -> impl Decoder { +pub(crate) fn mk_decoder(r: RawEntryReader) -> impl Decoder { Deflate64Decoder::new(r) } diff --git a/src/reader/sync/entry_reader/deflate_dec.rs b/src/reader/sync/entry_reader/deflate_dec.rs index afa1068..dfa6495 100644 --- a/src/reader/sync/entry_reader/deflate_dec.rs +++ b/src/reader/sync/entry_reader/deflate_dec.rs @@ -2,7 +2,7 @@ use std::io::Read; use flate2::read::DeflateDecoder; -use crate::reader::sync::{Decoder, LimitedReader}; +use crate::reader::sync::{Decoder, RawEntryReader}; impl Decoder for DeflateDecoder where @@ -17,6 +17,6 @@ where } } -pub(crate) fn mk_decoder(r: LimitedReader) -> impl Decoder { +pub(crate) fn mk_decoder(r: RawEntryReader) -> impl Decoder { DeflateDecoder::new(r) } diff --git a/src/reader/sync/entry_reader/lzma_dec.rs b/src/reader/sync/entry_reader/lzma_dec.rs index eb9d687..1898ebf 100644 --- a/src/reader/sync/entry_reader/lzma_dec.rs +++ b/src/reader/sync/entry_reader/lzma_dec.rs @@ -2,7 +2,7 @@ use lzma_rs::decompress::Stream; use std::io::{Read, Write}; use crate::{ - reader::sync::{Decoder, LimitedReader}, + reader::sync::{Decoder, RawEntryReader}, Error, UnsupportedError, }; @@ -102,9 +102,9 @@ where } pub(crate) fn mk_decoder( - mut r: LimitedReader, + mut r: RawEntryReader, uncompressed_size: u64, -) -> std::io::Result> { +) -> std::io::Result> { use byteorder::{LittleEndian, ReadBytesExt}; // see `appnote.txt` section 5.8 diff --git a/src/reader/sync/entry_reader/mod.rs b/src/reader/sync/entry_reader/mod.rs index e8bd373..7c1f448 100644 --- a/src/reader/sync/entry_reader/mod.rs +++ b/src/reader/sync/entry_reader/mod.rs @@ -3,7 +3,7 @@ use crate::{ error::*, format::*, - reader::sync::decoder::{Decoder, EOFNormalizer, LimitedReader, StoreDecoder}, + reader::sync::decoder::{Decoder, EOFNormalizer, RawEntryReader, StoreDecoder}, transition, }; @@ -45,7 +45,7 @@ enum State { hasher: crc32fast::Hasher, uncompressed_size: u64, header: LocalFileHeaderRecord, - decoder: Box>, + decoder: Box>, }, ReadDataDescriptor { metrics: EntryReadMetrics, @@ -93,8 +93,8 @@ where transition!(self.state => (S::ReadLocalHeader { buffer }) { // allow unnecessary mut for some feature combinations #[allow(unused_mut)] - let mut limited_reader = LimitedReader::new(buffer, self.inner.compressed_size); - let decoder: Box> = self.get_decoder(limited_reader)?; + let mut limited_reader = RawEntryReader::new(buffer, self.inner.compressed_size); + let decoder: Box> = self.get_decoder(limited_reader)?; S::ReadData { hasher: crc32fast::Hasher::new(), @@ -253,7 +253,7 @@ impl EntryReader where R: io::Read, { - const DEFAULT_BUFFER_SIZE: usize = 8 * 1024; + const DEFAULT_BUFFER_SIZE: usize = 256 * 1024; pub fn new(entry: &StoredEntry, get_reader: F) -> Self where @@ -272,14 +272,14 @@ where fn get_decoder( &self, - #[allow(unused_mut)] mut limited_reader: LimitedReader, - ) -> Result>, Error> { - let decoder: Box> = match self.method { - Method::Store => Box::new(StoreDecoder::new(limited_reader)), + #[allow(unused_mut)] mut raw_r: RawEntryReader, + ) -> Result>, Error> { + let decoder: Box> = match self.method { + Method::Store => Box::new(StoreDecoder::new(raw_r)), Method::Deflate => { cfg_if! { if #[cfg(feature = "deflate")] { - Box::new(deflate_dec::mk_decoder(limited_reader)) + Box::new(deflate_dec::mk_decoder(raw_r)) } else { return Err(Error::method_not_enabled(self.method)); } @@ -288,7 +288,7 @@ where Method::Deflate64 => { cfg_if! { if #[cfg(feature = "deflate64")] { - Box::new(deflate64_dec::mk_decoder(limited_reader)) + Box::new(deflate64_dec::mk_decoder(raw_r)) } else { return Err(Error::method_not_enabled(self.method)); } @@ -297,7 +297,7 @@ where Method::Lzma => { cfg_if! { if #[cfg(feature = "lzma")] { - Box::new(lzma_dec::mk_decoder(limited_reader,self.inner.uncompressed_size)?) + Box::new(lzma_dec::mk_decoder(raw_r,self.inner.uncompressed_size)?) } else { return Err(Error::method_not_enabled(self.method)); } @@ -306,7 +306,7 @@ where Method::Bzip2 => { cfg_if! { if #[cfg(feature = "bzip2")] { - Box::new(bzip2_dec::mk_decoder(limited_reader)) + Box::new(bzip2_dec::mk_decoder(raw_r)) } else { return Err(Error::method_not_enabled(self.method)); } @@ -315,7 +315,7 @@ where Method::Zstd => { cfg_if! { if #[cfg(feature = "zstd")] { - Box::new(zstd_dec::mk_decoder(limited_reader)?) + Box::new(zstd_dec::mk_decoder(raw_r)?) } else { return Err(Error::method_not_enabled(self.method)); } diff --git a/src/reader/sync/entry_reader/zstd_dec.rs b/src/reader/sync/entry_reader/zstd_dec.rs index 4fce919..786cfc3 100644 --- a/src/reader/sync/entry_reader/zstd_dec.rs +++ b/src/reader/sync/entry_reader/zstd_dec.rs @@ -1,23 +1,23 @@ -use std::io::{BufReader, Read}; +use std::io::{BufRead, Read}; use zstd::stream::Decoder as ZstdDecoder; -use crate::reader::sync::{Decoder, LimitedReader}; +use crate::reader::sync::{Decoder, RawEntryReader}; -impl Decoder for ZstdDecoder<'static, BufReader> +impl Decoder for ZstdDecoder<'static, R> where - R: Read, + R: Read + BufRead, { fn into_inner(self: Box) -> R { - Self::finish(*self).into_inner() + Self::finish(*self) } fn get_mut(&mut self) -> &mut R { - Self::get_mut(self).get_mut() + Self::get_mut(self) } } -pub(crate) fn mk_decoder(r: LimitedReader) -> std::io::Result> { - // TODO: have LimitedReader (and Buffer) implement BufRead, cf. https://github.com/fasterthanlime/rc-zip/issues/55 - ZstdDecoder::with_buffer(BufReader::new(r)) +pub(crate) fn mk_decoder(r: RawEntryReader) -> std::io::Result> { + tracing::trace!("Creating ZstdDecoder"); + ZstdDecoder::with_buffer(r) } diff --git a/src/reader/sync/read_zip.rs b/src/reader/sync/read_zip.rs index 18e22e6..01db41e 100644 --- a/src/reader/sync/read_zip.rs +++ b/src/reader/sync/read_zip.rs @@ -144,6 +144,7 @@ where { /// Returns a reader for the entry. pub fn reader(&self) -> EntryReader<::Cursor<'a>> { + tracing::trace!("Creating EntryReader"); EntryReader::new(self.entry, |offset| self.file.cursor_at(offset)) } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index a8657ab..b7741b7 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1,5 +1,3 @@ -use tracing_test::traced_test; - use chrono::{ offset::{FixedOffset, Utc}, DateTime, TimeZone, Timelike, @@ -324,8 +322,7 @@ fn test_cases() -> Vec { ] } -#[test] -#[traced_test] +#[test_log::test] fn read_from_slice() { let bytes = std::fs::read(zips_dir().join("test.zip")).unwrap(); let slice = &bytes[..]; @@ -333,25 +330,22 @@ fn read_from_slice() { assert_eq!(archive.entries().count(), 2); } -#[test] -#[traced_test] +#[test_log::test] fn read_from_file() { let f = File::open(zips_dir().join("test.zip")).unwrap(); let archive = f.read_zip().unwrap(); assert_eq!(archive.entries().count(), 2); } -#[test] -#[traced_test] +#[test_log::test] fn real_world_files() { for case in test_cases() { - eprintln!("============ testing {}", case.name()); + tracing::trace!("============ testing {}", case.name()); case.check(case.bytes().read_zip()); } } -#[test] -#[traced_test] +#[test_log::test] fn state_machine() { use rc_zip::reader::{ArchiveReader, ArchiveReaderResult};