diff --git a/brro-compressor/src/main.rs b/brro-compressor/src/main.rs index fd45419..a1c0d27 100644 --- a/brro-compressor/src/main.rs +++ b/brro-compressor/src/main.rs @@ -36,6 +36,7 @@ fn process_directory(arguments: &Args) -> Result<(), Box> { for entry in std::fs::read_dir(arguments.input.clone())? { let path = entry?.path(); if path.is_file() { + // We need to make sure we skip anything but BRO and WBRO, this can be done on single file processors process_single_file(path, arguments)?; } } diff --git a/brro-compressor/src/optimizer/mod.rs b/brro-compressor/src/optimizer/mod.rs index 76df29f..2d7d224 100644 --- a/brro-compressor/src/optimizer/mod.rs +++ b/brro-compressor/src/optimizer/mod.rs @@ -2,6 +2,8 @@ use log::debug; use types::metric_tag::MetricTag; use crate::{types, utils::{prev_power_of_two, f64_to_u64}, compressor::Compressor}; +pub mod utils; + /// Max Frame size, this can aprox. 36h of data at 1point/sec rate, a little more than 1 week at 1point/5sec /// and 1 month (30 days) at 1 point/20sec. /// This would be aprox. 1MB of Raw data (131072 * 64bits). diff --git a/brro-compressor/src/utils/writers/wav_writer.rs b/brro-compressor/src/optimizer/utils.rs similarity index 65% rename from brro-compressor/src/utils/writers/wav_writer.rs rename to brro-compressor/src/optimizer/utils.rs index 3a0541c..82f195f 100644 --- a/brro-compressor/src/utils/writers/wav_writer.rs +++ b/brro-compressor/src/optimizer/utils.rs @@ -1,25 +1,5 @@ -use std::path::PathBuf; -use hound::{WavSpec, WavWriter}; -use log::info; +use log::debug; -// Function to create a streaming writer for a file -pub fn write_optimal_wav(mut path: PathBuf, data: Vec, channels: i32) { - let (bitdepth, dc, _fractional) = analyze_data(&data); - // Make DC a float for operations - let fdc = dc as f64; - let header: WavSpec = generate_wav_header(Some(channels), bitdepth as u16, 8000); - path.set_extension("wav"); - let file = std::fs::OpenOptions::new().write(true).create(true).read(true).open(path).unwrap(); - let mut wav_writer = WavWriter::new(file, header).unwrap(); - for sample in data { - let _ = match bitdepth { - 8 => wav_writer.write_sample(as_i8(sample-fdc)), - 16 => wav_writer.write_sample(as_i16(sample-fdc)), - _ => wav_writer.write_sample(as_i32(sample-fdc)) - }; - } - let _ = wav_writer.finalize(); -} fn as_i8(value: f64) -> i8 { split_n(value).0 as i8 } @@ -71,25 +51,6 @@ fn split_n(x: f64) -> (i64, f64) { } } -fn join_u16_into_f64(bits: [u16; 4]) -> f64 { - let u64_bits = (bits[0] as u64) | - ((bits[1] as u64) << 16) | - ((bits[2] as u64) << 32) | - ((bits[3] as u64) << 48); - - - f64::from_bits(u64_bits) -} -fn generate_wav_header(channels: Option, bitdepth: u16, samplerate: u32) -> WavSpec { - - hound::WavSpec { - channels: channels.unwrap_or(4) as u16, - // TODO: Sample rate adaptations - sample_rate: samplerate, - bits_per_sample: bitdepth, - sample_format: hound::SampleFormat::Int - } -} fn analyze_data(data: &Vec) -> (i32, i64, bool) { let mut min: f64 = 0.0; let mut max: f64 = 0.0; @@ -112,9 +73,9 @@ fn analyze_data(data: &Vec) -> (i32, i64, bool) { // Finding the bitdepth without the DC component let recommended_bitdepth = find_bitdepth(max_int-min_int, min_int); if !fractional { - info!(" Recommended Bitdepth: {} ", recommended_bitdepth); + debug!(" Recommended Bitdepth: {} ", recommended_bitdepth); } else { - info!(" Fractional, Recommended Bitdepth: {}, Fractions max: {}", recommended_bitdepth, max_frac); + debug!(" Fractional, Recommended Bitdepth: {}, Fractions max: {}", recommended_bitdepth, max_frac); } (recommended_bitdepth, min_int, fractional) } @@ -135,6 +96,5 @@ fn find_bitdepth(max_int: i64, min_int: i64) -> i32 { _ => 64 }; - bitdepth.max(bitdepth_signed) } \ No newline at end of file diff --git a/brro-compressor/src/utils/file_type_detector.rs b/brro-compressor/src/utils/file_type_detector.rs deleted file mode 100644 index 964ce06..0000000 --- a/brro-compressor/src/utils/file_type_detector.rs +++ /dev/null @@ -1,28 +0,0 @@ -use std::fs::File; -use std::io::{BufReader, Read, Result, Take, Error, ErrorKind}; - -enum FileType { - Wav, - Raw, -} - -fn is_wav(reader: &mut Take>) -> Result { - let mut buffer = [0u8; 12]; - reader.read_exact(&mut buffer)?; - Ok(&buffer[..4] == b"RIFF" && &buffer[8..] == b"WAVE") -} - -fn detect_file_type(filename: &str) -> Result { - let file = File::open(filename)?; - let file_len = file.metadata()?.len(); - - if file_len < 12 { - return Err(Error::new(ErrorKind::InvalidData, "File is too short")); - } - - if is_wav(&mut BufReader::new(file).take(12))? { - Ok(FileType::Wav) - } else { - Ok(FileType::Raw) - } -} \ No newline at end of file diff --git a/brro-compressor/src/utils/mod.rs b/brro-compressor/src/utils/mod.rs index c4f2e84..ca6d157 100644 --- a/brro-compressor/src/utils/mod.rs +++ b/brro-compressor/src/utils/mod.rs @@ -1,9 +1,6 @@ pub mod error; pub mod writers; pub mod readers; - -mod file_type_detector; - // Is this the right place? pub fn prev_power_of_two(n: usize) -> usize { // n = 0 gives highest_bit_set_idx = 0. diff --git a/brro-compressor/src/utils/readers/mod.rs b/brro-compressor/src/utils/readers/mod.rs index 3228e92..63a578e 100644 --- a/brro-compressor/src/utils/readers/mod.rs +++ b/brro-compressor/src/utils/readers/mod.rs @@ -1,2 +1 @@ -pub mod bro_reader; -pub mod wav_reader; \ No newline at end of file +pub mod bro_reader; \ No newline at end of file diff --git a/brro-compressor/src/utils/readers/wav_reader.rs b/brro-compressor/src/utils/readers/wav_reader.rs deleted file mode 100644 index df8b5ec..0000000 --- a/brro-compressor/src/utils/readers/wav_reader.rs +++ /dev/null @@ -1,174 +0,0 @@ -// Implement a streaming reader here -use std::fs; -use std::io::{self, Error, Read}; -use std::path::Path; -use log::debug; -use regex::Regex; -use types::metric_tag::MetricTag; -use crate::types; - - -// Function to check if a file is a WAV file -fn is_wav_file(file_path: &Path) -> io::Result { - // Open the file for reading and read the first 12 bytes (header) of the file - let mut file = fs::File::open(file_path)?; - let mut header = [0u8; 12]; - file.read_exact(&mut header)?; - - // Check if the file starts with "RIFF" and ends with "WAVE" in the header - Ok(header.starts_with(b"RIFF") && &header[8..12] == b"WAVE") -} - -// Function to process a WAV file -fn process_wav_file(file_path: &Path) -> io::Result, MetricTag)>> { - let full_path_str = file_path.to_str().unwrap_or(""); - debug!("File: {} ,", full_path_str); - let wav_data = read_metrics_from_wav(full_path_str); - debug!("Data Len: {}", wav_data.len()); - // Depending on Metric Tag, apply a transformation - let tag = tag_metric(full_path_str); - Ok(Some((wav_data, tag))) -} - -// Function to process a RAW file -fn process_raw_file(file_path: &Path) -> io::Result, MetricTag)>> { - log::error!("Raw files not supported yet. File path: {:?}", file_path.display()); - Ok(None) -} - -/// Read a file by chunks and processes the chunks -pub fn process_by_chunk(file_path: &Path) -> Result<(), std::io::Error> { - let mut file = std::fs::File::open(file_path)?; - - let mut list_of_chunks = Vec::new(); - // 64KB at a time, assuming 64Bit samples, ~1024 samples. - let chunk_size = 0x10000; - - loop { - let mut chunk = Vec::with_capacity(chunk_size); - let n = file.by_ref().take(chunk_size as u64).read_to_end(&mut chunk)?; - if n == 0 { break; } - list_of_chunks.push(chunk); - if n < chunk_size { break; } - } - Ok(()) -} - - -pub fn read_file(file_path: &Path) -> Result, MetricTag)>, Error> { - if is_wav_file(file_path)? { - // If it's a WAV file, process it using the process_wav_file function - process_wav_file(file_path) - } else { - // If it's not a WAV file, process it as a RAW file using the process_raw_file function - process_raw_file(file_path) - } -} -/* -Reads a WAV file, checks the channels and the information contained there. From that -information takes a decision on the best channel, block size and bitrate for the BRRO -encoders. -*/ -fn tag_metric(filename: &str) -> MetricTag { - // Should sort this by the probability of each tag, so the ones that are more common are dealt first - // If it says percent_ or _utilization - let mut regex = Regex::new(r"(?m)percent_|_utilization").unwrap(); - if regex.captures(filename).is_some() { - // 2 significant digits resolution (Linux resolution) - return MetricTag::Percent(100); - } - // if it says _client_request - regex = Regex::new(r"(?m)_client_request").unwrap(); - if regex.captures(filename).is_some() { - // Fractional requests are nothing but an averaging artifact - return MetricTag::NotFloat; - } - // if it says _seconds - regex = Regex::new(r"(?m)_seconds").unwrap(); - if regex.captures(filename).is_some() { - // 1 micro second resolution - return MetricTag::Duration(1_000_000); - } - // if it says _seconds - regex = Regex::new(r"(?m)_networkindelta|_networkoutdelta|_heapmemoryused_").unwrap(); - if regex.captures(filename).is_some() { - return MetricTag::QuasiRandom; - } - MetricTag::Other -} - -fn read_metrics_from_wav(filename: &str) -> Vec { - let r_reader = hound::WavReader::open(filename); - let mut reader = match r_reader { - Ok(reader) => reader, - Err(_err) => { return Vec::new(); } - }; - let num_channels = reader.spec().channels as usize; - - let mut raw_data: Vec = Vec::new(); - let mut u64_holder: [u16; 4] = [0, 0, 0, 0]; - - // Iterate over the samples and channels and push each sample to the vector - let mut current_channel: usize = 0; - for sample in reader.samples::() { - u64_holder[current_channel] = sample.unwrap() as u16; - current_channel += 1; - if current_channel == num_channels { - raw_data.push(join_u16_into_f64(u64_holder)); - current_channel = 0; - } - } - raw_data -} - -fn join_u16_into_f64(bits: [u16; 4]) -> f64 { - let u64_bits = (bits[0] as u64) | - ((bits[1] as u64) << 16) | - ((bits[2] as u64) << 32) | - ((bits[3] as u64) << 48); - - - f64::from_bits(u64_bits) -} - -// Import the testing module -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_wav_file_with_wav() { - // Create a temporary file with a valid WAV header - let temp_file_path = "test.wav"; - let header: [u8; 12] = [82, 73, 70, 70, 4, 0, 0, 0, 87, 65, 86, 69]; - std::fs::write(temp_file_path, header).expect("Failed to create temporary WAV file"); - - // Check if the file is recognized as a WAV file - let path = Path::new(temp_file_path); - let result = is_wav_file(path); - - // Assert that it should be recognized as a WAV file - assert!(result.is_ok() && result.unwrap()); - - // Clean up the temporary file - std::fs::remove_file(temp_file_path).expect("Failed to remove temporary file"); - } - - #[test] - fn test_is_wav_file_with_non_wav() { - // Create a temporary file with a non-WAV header - let temp_file_path = "test.txt"; - let header: [u8; 12] = [84, 69, 83, 84, 32, 70, 73, 76, 69, 33, 33, 33]; - std::fs::write(temp_file_path, header).expect("Failed to create temporary non-WAV file"); - - // Check if the file is recognized as a WAV file - let path = Path::new(temp_file_path); - let result = is_wav_file(path); - - // Assert that it should not be recognized as a WAV file - assert!(result.is_ok() && !result.unwrap()); - - // Clean up the temporary file - std::fs::remove_file(temp_file_path).expect("Failed to remove temporary file"); - } -} \ No newline at end of file diff --git a/brro-compressor/src/utils/writers/mod.rs b/brro-compressor/src/utils/writers/mod.rs index fd30b82..e69de29 100644 --- a/brro-compressor/src/utils/writers/mod.rs +++ b/brro-compressor/src/utils/writers/mod.rs @@ -1 +0,0 @@ -pub mod wav_writer; diff --git a/wavbrro/src/wavbrro.rs b/wavbrro/src/wavbrro.rs index ad670f2..c9ad206 100644 --- a/wavbrro/src/wavbrro.rs +++ b/wavbrro/src/wavbrro.rs @@ -58,11 +58,13 @@ impl WavBrro { } // Receives a slice of f64 and writes in it's internal structure - fn load_slice(&mut self, data: &[f64]) { - let size = data.len(); - let inner = data.chunks(MAX_CHUNK_SIZE).map(|s| s.into()).collect(); - self.chunks = inner; - self.sample_count = size as u32; + fn from_slice(data: &[f64]) -> Self { + let sample_count = data.len(); + WavBrro { + sample_count: sample_count as u32, + bitdepth: 5, + chunks: data.chunks(MAX_CHUNK_SIZE).map(|s| s.into()).collect() + } } pub fn add_sample(&mut self, sample: f64) { @@ -91,8 +93,7 @@ impl WavBrro { // TODO: This will panic left and right, make it right pub fn to_file_with_data(file_path: &Path, data: &[f64]) { - let mut wb = WavBrro::new(); - wb.load_slice(data); + let wb = WavBrro::from_slice(data); let bytes = wb.to_bytes(); write_wavbrro_file(file_path, &bytes); }