diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml index 76ae3c2..5ab6fe2 100644 --- a/.github/workflows/rust-dev.yml +++ b/.github/workflows/rust-dev.yml @@ -23,13 +23,13 @@ jobs: cargo clean - name: Test with cargo run: | - RUST_BACKTRACE=full cargo test + RUST_BACKTRACE=full cargo test --features "experimental_index" - name: Check with clippy run: | - cargo clippy -- -W clippy::pedantic + cargo clippy --features "experimental_index" - name: Build release run: | - cargo build --release + cargo build --release --features "experimental_index" build-macos: @@ -45,10 +45,10 @@ jobs: cargo clean - name: Test with cargo run: | - RUST_BACKTRACE=full cargo test + RUST_BACKTRACE=full cargo test --features "experimental_index" - name: Check with clippy run: | - cargo clippy -- -W clippy::pedantic + cargo clippy --features "experimental_index" - name: Build release run: | - cargo build --release + cargo build --release --features "experimental_index" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4431fb8..bb91cc0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -27,11 +27,11 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release + cargo build --release --features "experimental_index" cargo clean - name: Test with cargo run: | - cargo test + cargo test --features "experimental_index" cargo clean - name: Benchmark with criterion run: | @@ -53,11 +53,11 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release + cargo build --release --features "experimental_index" cargo clean - name: Test with cargo run: | - cargo test + cargo test --features "experimental_index" cargo clean - name: Benchmark with criterion run: | diff --git a/Cargo.toml b/Cargo.toml index c89ccc2..7c51d5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" -version = "0.8.0" +version = "0.9.0" readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] @@ -18,7 +18,7 @@ edition = "2021" exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] [dependencies] -eccodes-sys = "0.5.1" +eccodes-sys = "0.5.2" libc = "0.2" thiserror = "1.0" bytes = "1.5" @@ -27,20 +27,26 @@ errno = "0.3" num-derive = "0.4.1" num-traits = "0.2" fallible-iterator = "0.3" +fallible-streaming-iterator = "0.1.9" [dev-dependencies] -eccodes-sys = "0.5.1" reqwest = { version = "0.11", features = ["rustls-tls"] } tokio = { version = "1.35", features = ["macros", "rt"] } criterion = "0.5" testing_logger = "0.1" +rand = "0.8" +anyhow = "1.0" [features] docs = ["eccodes-sys/docs"] +experimental_index = [] [package.metadata.docs.rs] -features = ["docs"] +features = ["docs", "experimental_index"] [[bench]] name = "main" harness = false + +[lib] +doctest = false diff --git a/benches/main.rs b/benches/main.rs index 61c9e72..6445e86 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,4 +1,4 @@ -use eccodes::FallibleIterator; +use eccodes::FallibleStreamingIterator; use std::path::Path; use criterion::{black_box, criterion_group, criterion_main, Criterion}; diff --git a/data/iceland-surface.idx b/data/iceland-surface.idx new file mode 100644 index 0000000..2aae480 Binary files /dev/null and b/data/iceland-surface.idx differ diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index e4c4d57..ec296fe 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,14 +1,16 @@ -use eccodes_sys::codes_handle; -use fallible_iterator::FallibleIterator; +use std::ptr; + +use fallible_streaming_iterator::FallibleStreamingIterator; use crate::{ codes_handle::{CodesHandle, KeyedMessage}, errors::CodesError, - intermediate_bindings::{ - codes_get_message_copy, codes_handle_delete, codes_handle_new_from_file, - codes_handle_new_from_message_copy, - }, + intermediate_bindings::{codes_handle_delete, codes_handle_new_from_file}, }; +#[cfg(feature = "experimental_index")] +use crate::{intermediate_bindings::codes_index::codes_handle_new_from_index, CodesIndex}; + +use super::GribFile; ///`FallibleIterator` implementation for `CodesHandle` to access GRIB messages inside file. /// @@ -18,7 +20,7 @@ use crate::{ ///Therefore this crate utilizes the `Iterator` to provide the access to GRIB messages in ///a safe and convienient way. /// -///[`FallibleIterator`](fallible_iterator::FallibleIterator) is used instead of classic `Iterator` +///[`FallibleIterator`] is used instead of classic `Iterator` ///because internal ecCodes functions can return error codes when the GRIB file ///is corrupted and for some other reasons. The usage of `FallibleIterator` is sligthly different ///than usage of `Iterator`, check its documentation for more details. @@ -77,71 +79,120 @@ use crate::{ ///## Errors ///The `next()` method will return [`CodesInternal`](crate::errors::CodesInternal) ///when internal ecCodes function returns non-zero code. -impl FallibleIterator for CodesHandle { +impl FallibleStreamingIterator for CodesHandle { type Item = KeyedMessage; type Error = CodesError; - fn next(&mut self) -> Result, Self::Error> { - let file_handle; + fn advance(&mut self) -> Result<(), Self::Error> { unsafe { - codes_handle_delete(self.file_handle)?; - file_handle = codes_handle_new_from_file(self.file_pointer, self.product_kind); + codes_handle_delete(self.unsafe_message.message_handle)?; } - match file_handle { - Ok(h) => { - self.file_handle = h; + // nullify message handle so that destructor is harmless + // it might be excessive but it follows the correct pattern + self.unsafe_message.message_handle = ptr::null_mut(); - if self.file_handle.is_null() { - Ok(None) - } else { - let message = get_message_from_handle(h); - Ok(Some(message)) - } - } - Err(e) => Err(e), + let new_eccodes_handle = + unsafe { codes_handle_new_from_file(self.source.pointer, self.product_kind)? }; + + self.unsafe_message = KeyedMessage { + message_handle: new_eccodes_handle, + iterator_flags: None, + iterator_namespace: None, + keys_iterator: None, + keys_iterator_next_item_exists: false, + nearest_handle: None, + }; + + Ok(()) + } + + fn get(&self) -> Option<&Self::Item> { + if self.unsafe_message.message_handle.is_null() { + None + } else { + Some(&self.unsafe_message) } } } -fn get_message_from_handle(handle: *mut codes_handle) -> KeyedMessage { - let new_handle; - let new_buffer; +#[cfg(feature = "experimental_index")] +impl FallibleStreamingIterator for CodesHandle { + type Item = KeyedMessage; - unsafe { - new_buffer = codes_get_message_copy(handle).expect( - "Getting message clone failed. - Please report this panic on Github", - ); - new_handle = codes_handle_new_from_message_copy(&new_buffer); + type Error = CodesError; + + fn advance(&mut self) -> Result<(), Self::Error> { + unsafe { + codes_handle_delete(self.unsafe_message.message_handle)?; + } + + // nullify message handle so that destructor is harmless + // it might be excessive but it follows the correct pattern + self.unsafe_message.message_handle = ptr::null_mut(); + + let new_eccodes_handle = unsafe { codes_handle_new_from_index(self.source.pointer)? }; + + self.unsafe_message = KeyedMessage { + message_handle: new_eccodes_handle, + iterator_flags: None, + iterator_namespace: None, + keys_iterator: None, + keys_iterator_next_item_exists: false, + nearest_handle: None, + }; + + Ok(()) } - KeyedMessage { - message_handle: new_handle, - iterator_flags: None, - iterator_namespace: None, - keys_iterator: None, - keys_iterator_next_item_exists: false, - nearest_handle: None, + fn get(&self) -> Option<&Self::Item> { + if self.unsafe_message.message_handle.is_null() { + None + } else { + Some(&self.unsafe_message) + } } } #[cfg(test)] mod tests { - use crate::codes_handle::{CodesHandle, KeyType, KeyedMessage, ProductKind}; - use crate::FallibleIterator; + use crate::codes_handle::{CodesHandle, KeyType, ProductKind}; + use anyhow::Result; + use fallible_streaming_iterator::FallibleStreamingIterator; use std::path::Path; #[test] - fn iterator_fn() { + fn iterator_lifetimes() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + + let msg1 = handle.next()?.unwrap(); + let key1 = msg1.read_key("typeOfLevel")?; + + let msg2 = handle.next()?.unwrap(); + let key2 = msg2.read_key("typeOfLevel")?; + + let msg3 = handle.next()?.unwrap(); + let key3 = msg3.read_key("typeOfLevel")?; + + assert_eq!(key1.value, KeyType::Str("isobaricInhPa".to_string())); + assert_eq!(key2.value, KeyType::Str("isobaricInhPa".to_string())); + assert_eq!(key3.value, KeyType::Str("isobaricInhPa".to_string())); + + Ok(()) + } + + #[test] + fn iterator_fn() -> Result<()> { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - while let Some(msg) = handle.next().unwrap() { - let key = msg.read_key("shortName").unwrap(); + while let Some(msg) = handle.next()? { + let key = msg.read_key("shortName")?; match key.value { KeyType::Str(_) => {} @@ -149,9 +200,20 @@ mod tests { } } - let handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + Ok(()) + } - let handle_collected: Vec = handle.collect().unwrap(); + #[test] + fn iterator_collected() -> Result<()> { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + + let mut handle_collected = vec![]; + + while let Some(msg) = handle.next()? { + handle_collected.push(msg.clone()); + } for msg in handle_collected { let key = msg.read_key("name").unwrap(); @@ -160,6 +222,8 @@ mod tests { _ => panic!("Incorrect variant of string key"), } } + + Ok(()) } #[test] @@ -178,24 +242,23 @@ mod tests { } #[test] - fn iterator_collect() { + fn iterator_filter() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); // Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface" // First, filter and collect the messages to get those that we want - let mut level: Vec = handle - .filter(|msg| { - Ok( - msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) - && msg.read_key("typeOfLevel")?.value - == KeyType::Str("surface".to_string()), - ) - }) - .collect() - .unwrap(); + let mut level = vec![]; + + while let Some(msg) = handle.next()? { + if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) + && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) + { + level.push(msg.clone()); + } + } // Now unwrap and access the first and only element of resulting vector // Find nearest modifies internal KeyedMessage fields so we need mutable reference @@ -211,5 +274,7 @@ mod tests { "value: {}, distance: {}", nearest_gridpoints[3].value, nearest_gridpoints[3].distance ); + + Ok(()) } } diff --git a/src/codes_handle/keyed_message/iterator.rs b/src/codes_handle/keyed_message/iterator.rs index 751c49f..18ff274 100644 --- a/src/codes_handle/keyed_message/iterator.rs +++ b/src/codes_handle/keyed_message/iterator.rs @@ -18,7 +18,7 @@ use super::KeysIteratorFlags; ///so it is probably more efficient to call that function directly only for keys you ///are interested in. /// -///[`FallibleIterator`](fallible_iterator::FallibleIterator) is used instead of classic `Iterator` +///[`FallibleIterator`] is used instead of classic `Iterator` ///because internal ecCodes functions can return internal error in some edge-cases. ///The usage of `FallibleIterator` is sligthly different than usage of `Iterator`, ///check its documentation for more details. @@ -156,17 +156,19 @@ impl KeyedMessage { #[cfg(test)] mod tests { + use anyhow::Result; + use crate::codes_handle::{CodesHandle, KeysIteratorFlags, ProductKind}; - use crate::FallibleIterator; + use crate::{FallibleIterator, FallibleStreamingIterator}; use std::path::Path; #[test] - fn keys_iterator_parameters() { + fn keys_iterator_parameters() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut current_message = handle.next()?.unwrap().clone(); assert!(current_message.iterator_flags.is_none()); assert!(current_message.iterator_namespace.is_none()); @@ -193,15 +195,17 @@ mod tests { while let Some(key) = current_message.next().unwrap() { assert!(!key.name.is_empty()); } + + Ok(()) } #[test] - fn invalid_namespace() { + fn invalid_namespace() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut current_message = handle.next()?.unwrap().clone(); let flags = vec![ KeysIteratorFlags::AllKeys, //0 @@ -214,5 +218,7 @@ mod tests { while let Some(key) = current_message.next().unwrap() { assert!(!key.name.is_empty()); } + + Ok(()) } } diff --git a/src/codes_handle/keyed_message/mod.rs b/src/codes_handle/keyed_message/mod.rs index 2ca6237..460143b 100644 --- a/src/codes_handle/keyed_message/mod.rs +++ b/src/codes_handle/keyed_message/mod.rs @@ -122,7 +122,7 @@ impl Drop for KeyedMessage { ///Technical note: delete functions in ecCodes can only fail with [`CodesInternalError`](crate::errors::CodesInternal::CodesInternalError) ///when other functions corrupt the inner memory of pointer, in that case memory leak is possible. ///In case of corrupt pointer segmentation fault will occur. - ///The pointers are cleared at the end of drop as they ar not not functional despite the result of delete functions. + ///The pointers are cleared at the end of drop as they are not functional despite the result of delete functions. fn drop(&mut self) { if let Some(nrst) = self.nearest_handle { unsafe { @@ -163,17 +163,18 @@ impl Drop for KeyedMessage { #[cfg(test)] mod tests { use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::FallibleIterator; + use crate::{FallibleIterator, FallibleStreamingIterator}; + use anyhow::Result; use std::path::Path; use testing_logger; #[test] - fn key_clone() { + fn key_clone() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap(); let cloned_message = current_message.clone(); assert_ne!( @@ -184,45 +185,50 @@ mod tests { assert!(cloned_message.iterator_namespace.is_none()); assert!(cloned_message.keys_iterator.is_none()); assert!(!cloned_message.keys_iterator_next_item_exists); + + Ok(()) } #[test] - fn message_drop() { + fn message_drop() -> Result<()> { testing_logger::setup(); let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut current_message = handle.next()?.unwrap().clone(); - let _key = current_message.next().unwrap().unwrap(); + let _key = current_message.next()?.unwrap(); drop(current_message); testing_logger::validate(|captured_logs| { assert_eq!(captured_logs.len(), 0); }); + + Ok(()) } #[test] - fn find_nearest() { + fn find_nearest() -> Result<()> { let file_path1 = Path::new("./data/iceland.grib"); let file_path2 = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; let mut handle1 = CodesHandle::new_from_file(file_path1, product_kind).unwrap(); - let mut msg1 = handle1.next().unwrap().unwrap(); + let msg1 = handle1.next()?.unwrap(); + let out1 = msg1.clone().find_nearest(64.13, -21.89).unwrap(); let mut handle2 = CodesHandle::new_from_file(file_path2, product_kind).unwrap(); - let mut msg2 = handle2.next().unwrap().unwrap(); - - let out1 = msg1.find_nearest(64.13, -21.89).unwrap(); - let out2 = msg2.find_nearest(64.13, -21.89).unwrap(); + let msg2 = handle2.next()?.unwrap(); + let out2 = msg2.clone().find_nearest(64.13, -21.89).unwrap(); assert!(out1[0].value > 10000.0); assert!(out2[3].index == 551); assert!(out1[1].lat == 64.0); assert!(out2[2].lon == -21.75); assert!(out1[0].distance > 15.0); + + Ok(()) } } diff --git a/src/codes_handle/keyed_message/read.rs b/src/codes_handle/keyed_message/read.rs index d11f828..1fcf994 100644 --- a/src/codes_handle/keyed_message/read.rs +++ b/src/codes_handle/keyed_message/read.rs @@ -165,12 +165,14 @@ impl KeyedMessage { #[cfg(test)] mod tests { + use anyhow::Result; + use crate::codes_handle::{CodesHandle, KeyType, ProductKind}; - use crate::FallibleIterator; + use crate::{FallibleIterator, FallibleStreamingIterator}; use std::path::Path; #[test] - fn key_reader() { + fn key_reader() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; @@ -217,66 +219,76 @@ mod tests { } assert_eq!(double_arr_key.name, "values"); + + Ok(()) } #[test] - fn era5_keys() { + fn era5_keys() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut current_message = handle.next()?.unwrap().clone(); for i in 0..=300 { let key = current_message.next(); println!("{}: {:?}", i, key); } + + Ok(()) } #[test] - fn gfs_keys() { + fn gfs_keys() -> Result<()> { let file_path = Path::new("./data/gfs.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut current_message = handle.next()?.unwrap().clone(); for i in 0..=300 { let key = current_message.next(); println!("{}: {:?}", i, key); } + + Ok(()) } #[test] - fn missing_key() { + fn missing_key() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap(); let missing_key = current_message.read_key("doesNotExist"); assert!(missing_key.is_err()); + + Ok(()) } #[test] - fn benchmark_keys() { + fn benchmark_keys() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg = handle.next().unwrap().unwrap(); + let msg = handle.next()?.unwrap(); - let _ = msg.read_key("dataDate").unwrap(); - let _ = msg.read_key("jDirectionIncrementInDegrees").unwrap(); - let _ = msg.read_key("values").unwrap(); - let _ = msg.read_key("name").unwrap(); - let _ = msg.read_key("section1Padding").unwrap(); - let _ = msg.read_key("experimentVersionNumber").unwrap(); + let _ = msg.read_key("dataDate")?; + let _ = msg.read_key("jDirectionIncrementInDegrees")?; + let _ = msg.read_key("values")?; + let _ = msg.read_key("name")?; + let _ = msg.read_key("section1Padding")?; + let _ = msg.read_key("experimentVersionNumber")?; let _ = msg .read_key("zero") .unwrap_or_else(|_| msg.read_key("zeros").unwrap()); + + Ok(()) } } diff --git a/src/codes_handle/keyed_message/write.rs b/src/codes_handle/keyed_message/write.rs index c8a142e..b33ee4e 100644 --- a/src/codes_handle/keyed_message/write.rs +++ b/src/codes_handle/keyed_message/write.rs @@ -134,120 +134,128 @@ impl KeyedMessage { #[cfg(test)] mod tests { + use anyhow::{Ok, Result}; + use crate::{ codes_handle::{ CodesHandle, Key, KeyType::{self}, ProductKind, }, - FallibleIterator, + FallibleStreamingIterator, }; use std::{fs::remove_file, path::Path}; #[test] - fn write_message() { + fn write_message_ref() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); - - drop(handle); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap(); let out_path = Path::new("./data/iceland_write.grib"); - current_message.write_to_file(out_path, false).unwrap(); + current_message.write_to_file(out_path, false)?; + + remove_file(out_path)?; - remove_file(out_path).unwrap(); + Ok(()) } #[test] - fn write_message_clone() { + fn write_message_clone() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap().clone(); drop(handle); let out_path = Path::new("./data/iceland_write_clone.grib"); - current_message.write_to_file(out_path, false).unwrap(); + current_message.write_to_file(out_path, false)?; + + remove_file(out_path)?; - remove_file(out_path).unwrap(); + Ok(()) } #[test] - fn append_message() { + fn append_message() -> Result<()> { let product_kind = ProductKind::GRIB; let out_path = Path::new("./data/iceland_append.grib"); let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); - current_message.write_to_file(out_path, false).unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap(); + current_message.write_to_file(out_path, false)?; let file_path = Path::new("./data/iceland-levels.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); - current_message.write_to_file(out_path, true).unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap(); + current_message.write_to_file(out_path, true)?; - remove_file(out_path).unwrap(); + remove_file(out_path)?; + + Ok(()) } #[test] - fn write_key() { + fn write_key() -> Result<()> { let product_kind = ProductKind::GRIB; let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut current_message = handle.next()?.unwrap().clone(); - let old_key = current_message.read_key("centre").unwrap(); + let old_key = current_message.read_key("centre")?; let new_key = Key { name: "centre".to_string(), value: KeyType::Str("cnmc".to_string()), }; - current_message.write_key(new_key.clone()).unwrap(); + current_message.write_key(new_key.clone())?; - let read_key = current_message.read_key("centre").unwrap(); + let read_key = current_message.read_key("centre")?; assert_eq!(new_key, read_key); assert_ne!(old_key, read_key); + + Ok(()) } #[test] - fn edit_keys_and_save() { + fn edit_keys_and_save() -> Result<()> { let product_kind = ProductKind::GRIB; let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let mut current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut current_message = handle.next()?.unwrap().clone(); - let old_key = current_message.read_key("centre").unwrap(); + let old_key = current_message.read_key("centre")?; let new_key = Key { name: "centre".to_string(), value: KeyType::Str("cnmc".to_string()), }; - current_message.write_key(new_key.clone()).unwrap(); + current_message.write_key(new_key.clone())?; - current_message - .write_to_file(Path::new("./data/iceland_edit.grib"), false) - .unwrap(); + current_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; let file_path = Path::new("./data/iceland_edit.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let current_message = handle.next().unwrap().unwrap(); + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let current_message = handle.next()?.unwrap(); - let read_key = current_message.read_key("centre").unwrap(); + let read_key = current_message.read_key("centre")?; assert_eq!(new_key, read_key); assert_ne!(old_key, read_key); - remove_file(Path::new("./data/iceland_edit.grib")).unwrap(); + remove_file(Path::new("./data/iceland_edit.grib"))?; + + Ok(()) } } diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 1c980cc..66e2a03 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -1,13 +1,16 @@ //!Main crate module containing definition of `CodesHandle` //!and all associated functions and data structures -use crate::errors::CodesError; +#[cfg(feature = "experimental_index")] +use crate::{codes_index::CodesIndex, intermediate_bindings::codes_index::codes_index_delete}; +use crate::CodesError; use bytes::Bytes; use eccodes_sys::{codes_handle, codes_keys_iterator, codes_nearest, ProductKind_PRODUCT_GRIB}; use errno::errno; use libc::{c_char, c_void, size_t, FILE}; use log::warn; use std::{ + fmt::Debug, fs::{File, OpenOptions}, os::unix::prelude::AsRawFd, path::Path, @@ -24,15 +27,21 @@ use eccodes_sys::{ mod iterator; mod keyed_message; +#[derive(Debug)] +#[doc(hidden)] +pub struct GribFile { + pointer: *mut FILE, +} + ///Main structure used to operate on the GRIB file. ///It takes a full ownership of the accessed file. ///It can be constructed either using a file or a memory buffer. #[derive(Debug)] -pub struct CodesHandle { - file_handle: *mut codes_handle, +pub struct CodesHandle { _data: DataContainer, - file_pointer: *mut FILE, + source: SOURCE, product_kind: ProductKind, + unsafe_message: KeyedMessage, } ///Structure used to access keys inside the GRIB file message. @@ -109,6 +118,8 @@ pub enum KeysIteratorFlags { enum DataContainer { FileBytes(Bytes), FileBuffer(File), + #[cfg(feature = "experimental_index")] + Empty(), } ///Enum representing the kind of product (file type) inside handled file. @@ -134,7 +145,7 @@ pub struct NearestGridpoint { pub value: f64, } -impl CodesHandle { +impl CodesHandle { ///The constructor that takes a [`path`](Path) to an existing file and ///a requested [`ProductKind`] and returns the [`CodesHandle`] object. /// @@ -168,7 +179,7 @@ impl CodesHandle { ///when the stream cannot be created from the file descriptor. /// ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`](eccodes_sys::codes_handle) cannot be created. + ///when internal [`codes_handle`] cannot be created. /// ///Returns [`CodesError::NoMessages`] when there is no message of requested type ///in the provided file. @@ -176,13 +187,20 @@ impl CodesHandle { let file = OpenOptions::new().read(true).open(file_path)?; let file_pointer = open_with_fdopen(&file)?; - let file_handle = null_mut(); - Ok(CodesHandle { _data: (DataContainer::FileBuffer(file)), - file_handle, - file_pointer, + source: GribFile { + pointer: file_pointer, + }, product_kind, + unsafe_message: KeyedMessage { + message_handle: null_mut(), + iterator_flags: None, + iterator_namespace: None, + keys_iterator: None, + keys_iterator_next_item_exists: false, + nearest_handle: None, + }, }) } @@ -220,7 +238,7 @@ impl CodesHandle { ///when the file stream cannot be created. /// ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`](eccodes_sys::codes_handle) cannot be created. + ///when internal [`codes_handle`] cannot be created. /// ///Returns [`CodesError::NoMessages`] when there is no message of requested type ///in the provided file. @@ -230,17 +248,49 @@ impl CodesHandle { ) -> Result { let file_pointer = open_with_fmemopen(&file_data)?; - let file_handle = null_mut(); - Ok(CodesHandle { _data: (DataContainer::FileBytes(file_data)), - file_handle, - file_pointer, + source: GribFile { + pointer: file_pointer, + }, product_kind, + unsafe_message: KeyedMessage { + message_handle: null_mut(), + iterator_flags: None, + iterator_namespace: None, + keys_iterator: None, + keys_iterator_next_item_exists: false, + nearest_handle: None, + }, }) } } +#[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +impl CodesHandle { + pub fn new_from_index( + index: CodesIndex, + product_kind: ProductKind, + ) -> Result { + let new_handle = CodesHandle { + _data: DataContainer::Empty(), //unused, index owns data + source: index, + product_kind, + unsafe_message: KeyedMessage { + message_handle: null_mut(), + iterator_flags: None, + iterator_namespace: None, + keys_iterator: None, + keys_iterator_next_item_exists: false, + nearest_handle: None, + }, + }; + + Ok(new_handle) + } +} + fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { let file_ptr; unsafe { @@ -275,37 +325,63 @@ fn open_with_fmemopen(file_data: &Bytes) -> Result<*mut FILE, CodesError> { Ok(file_ptr) } -impl Drop for CodesHandle { - ///Executes the destructor for this type. - ///This method calls `fclose()` from libc for graceful cleanup. - /// - ///Currently it is assumed that under normal circumstances this destructor never fails. - ///However in some edge cases fclose can return non-zero code. - ///In such case all pointers and file descriptors are safely deleted. - ///However memory leaks can still occur. - /// - ///If any function called in the destructor returns an error warning will appear in log. - ///If bugs occurs during `CodesHandle` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). - fn drop(&mut self) { +/// This trait is neccessary because (1) drop in GribFile/IndexFile cannot +/// be called directly as source cannot be moved out of shared reference +/// and (2) Drop drops fields in arbitrary order leading to fclose() failing +#[doc(hidden)] +pub trait SpecialDrop { + fn spec_drop(&mut self); +} + +impl SpecialDrop for GribFile { + fn spec_drop(&mut self) { //fclose() can fail in several different cases, however there is not much //that we can nor we should do about it. the promise of fclose() is that //the stream will be disassociated from the file after the call, therefore //use of stream after the call to fclose() is undefined behaviour, so we clear it let return_code; unsafe { - return_code = libc::fclose(self.file_pointer); - } - - if return_code != 0 { - let error_val = errno(); - warn!( + if !self.pointer.is_null() { + return_code = libc::fclose(self.pointer); + if return_code != 0 { + let error_val = errno(); + warn!( "fclose() returned an error and your file might not have been correctly saved. Error code: {}; Error message: {}", error_val.0, error_val ); + } + } + } + + self.pointer = null_mut(); + } +} + +#[cfg(feature = "experimental_index")] +impl SpecialDrop for CodesIndex { + fn spec_drop(&mut self) { + unsafe { + codes_index_delete(self.pointer); } - self.file_pointer = null_mut(); + self.pointer = null_mut(); + } +} + +impl Drop for CodesHandle { + ///Executes the destructor for this type. + ///This method calls `fclose()` from libc for graceful cleanup. + /// + ///Currently it is assumed that under normal circumstances this destructor never fails. + ///However in some edge cases fclose can return non-zero code. + ///In such case all pointers and file descriptors are safely deleted. + ///However memory leaks can still occur. + /// + ///If any function called in the destructor returns an error warning will appear in log. + ///If bugs occurs during `CodesHandle` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). + fn drop(&mut self) { + self.source.spec_drop(); } } @@ -314,6 +390,8 @@ mod tests { use eccodes_sys::ProductKind_PRODUCT_GRIB; use crate::codes_handle::{CodesHandle, DataContainer, ProductKind}; + #[cfg(feature = "experimental_index")] + use crate::codes_index::{CodesIndex, Select}; use log::Level; use std::path::Path; @@ -324,13 +402,13 @@ mod tests { let handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - assert!(!handle.file_pointer.is_null()); - assert!(handle.file_handle.is_null()); + assert!(!handle.source.pointer.is_null()); + assert!(handle.unsafe_message.message_handle.is_null()); assert_eq!(handle.product_kind as u32, { ProductKind_PRODUCT_GRIB }); let metadata = match &handle._data { - DataContainer::FileBytes(_) => panic!(), DataContainer::FileBuffer(file) => file.metadata().unwrap(), + _ => panic!(), }; println!("{:?}", metadata); @@ -349,16 +427,39 @@ mod tests { .unwrap(); let handle = CodesHandle::new_from_memory(file_data, product_kind).unwrap(); - assert!(!handle.file_pointer.is_null()); - assert!(handle.file_handle.is_null()); + assert!(!handle.source.pointer.is_null()); + assert!(handle.unsafe_message.message_handle.is_null()); assert_eq!(handle.product_kind as u32, { ProductKind_PRODUCT_GRIB }); match &handle._data { DataContainer::FileBytes(file) => assert!(!file.is_empty()), - DataContainer::FileBuffer(_) => panic!(), + _ => panic!(), }; } + #[test] + #[cfg(feature = "experimental_index")] + fn index_constructor_and_destructor() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + let i_ptr = index.pointer.clone(); + + let handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + + assert_eq!(handle.source.pointer, i_ptr); + assert!(handle.unsafe_message.message_handle.is_null()); + } + #[tokio::test] async fn codes_handle_drop() { testing_logger::setup(); diff --git a/src/codes_index/mod.rs b/src/codes_index/mod.rs new file mode 100644 index 0000000..1b79cb1 --- /dev/null +++ b/src/codes_index/mod.rs @@ -0,0 +1,160 @@ +//!Main crate module containing definition of `CodesIndex` +//!and all associated functions and data structures + +use crate::{ + codes_handle::SpecialDrop, + errors::CodesError, + intermediate_bindings::codes_index::{ + codes_index_add_file, codes_index_new, codes_index_read, codes_index_select_double, + codes_index_select_long, codes_index_select_string, + }, +}; +use eccodes_sys::codes_index; +use std::path::Path; + +#[derive(Debug)] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +pub struct CodesIndex { + pub(crate) pointer: *mut codes_index, +} +pub trait Select { + fn select(self, key: &str, value: T) -> Result; +} + +impl CodesIndex { + #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + pub fn new_from_keys(keys: &[&str]) -> Result { + let keys = keys.join(","); + + let index_handle; + unsafe { + // technically codes_index_new can also select keys + // but that would unnecessarily diverge the API + // and would be error prone + index_handle = codes_index_new(&keys)?; + } + Ok(CodesIndex { + pointer: index_handle, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + pub fn read_from_file(index_file_path: &Path) -> Result { + let file_path = index_file_path.to_str().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") + })?; + + let index_handle; + unsafe { + index_handle = codes_index_read(file_path)?; + } + + Ok(CodesIndex { + pointer: index_handle, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + pub fn add_grib_file(self, index_file_path: &Path) -> Result { + let file_path = index_file_path.to_str().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") + })?; + + let new_index = self; + + unsafe { + codes_index_add_file(new_index.pointer, file_path)?; + } + + Ok(new_index) + } +} + +impl Select for CodesIndex { + fn select(self, key: &str, value: i64) -> Result { + let new_index = self; + unsafe { + codes_index_select_long(new_index.pointer, key, value)?; + } + + Ok(new_index) + } +} +impl Select for CodesIndex { + fn select(self, key: &str, value: f64) -> Result { + let new_index = self; + unsafe { + codes_index_select_double(new_index.pointer, key, value)?; + } + Ok(new_index) + } +} +impl Select<&str> for CodesIndex { + fn select(self, key: &str, value: &str) -> Result { + let new_index = self; + unsafe { + codes_index_select_string(new_index.pointer, key, value)?; + } + Ok(new_index) + } +} + +impl Drop for CodesIndex { + fn drop(&mut self) { + self.spec_drop(); + } +} + +#[cfg(test)] +mod tests { + use crate::codes_index::{CodesIndex, Select}; + use std::path::Path; + #[test] + fn index_constructors() { + { + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + assert!(!index.pointer.is_null()); + } + { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path).unwrap(); + assert!(!index.pointer.is_null()); + } + } + + #[test] + fn index_destructor() { + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + + drop(index) + } + + #[test] + fn add_file() { + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + let grib_path = Path::new("./data/iceland.grib"); + let index = index.add_grib_file(grib_path).unwrap(); + + assert!(!index.pointer.is_null()); + } + + #[test] + fn index_selection() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + assert!(!index.pointer.is_null()); + } +} diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs new file mode 100644 index 0000000..c285810 --- /dev/null +++ b/src/intermediate_bindings/codes_index.rs @@ -0,0 +1,147 @@ +#![allow(non_camel_case_types)] +#![allow(clippy::module_name_repetitions)] + +use eccodes_sys::{codes_index, CODES_LOCK}; +use std::{ffi::CString, ptr}; + +#[cfg(target_os = "macos")] +type _SYS_IO_FILE = eccodes_sys::__sFILE; + +#[cfg(not(target_os = "macos"))] +type _SYS_IO_FILE = eccodes_sys::_IO_FILE; + +use eccodes_sys::{codes_context, codes_handle}; +use num_traits::FromPrimitive; + +use crate::errors::{CodesError, CodesInternal}; + +// all index functions are safeguarded by a lock +// because there are random errors appearing when using the index functions concurrently + +pub unsafe fn codes_index_new(keys: &str) -> Result<*mut codes_index, CodesError> { + let context: *mut codes_context = ptr::null_mut(); //default context + let mut error_code: i32 = 0; + let keys = CString::new(keys).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let codes_index = eccodes_sys::codes_index_new(context, keys.as_ptr(), &mut error_code); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(codes_index) +} + +pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, CodesError> { + let filename = CString::new(filename).unwrap(); + let context: *mut codes_context = ptr::null_mut(); //default context + let mut error_code: i32 = 0; + + let _g = CODES_LOCK.lock().unwrap(); + let codes_index = eccodes_sys::codes_index_read(context, filename.as_ptr(), &mut error_code); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(codes_index) +} + +pub unsafe fn codes_index_delete(index: *mut codes_index) { + if index.is_null() { + return; + } + + let _g = CODES_LOCK.lock().unwrap(); + eccodes_sys::codes_index_delete(index); +} + +pub unsafe fn codes_index_add_file( + index: *mut codes_index, + filename: &str, +) -> Result<(), CodesError> { + let filename = CString::new(filename).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_add_file(index, filename.as_ptr()); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_index_select_long( + index: *mut codes_index, + key: &str, + value: i64, +) -> Result<(), CodesError> { + let key = CString::new(key).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_select_long(index, key.as_ptr(), value); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_index_select_double( + index: *mut codes_index, + key: &str, + value: f64, +) -> Result<(), CodesError> { + let key = CString::new(key).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_select_double(index, key.as_ptr(), value); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_index_select_string( + index: *mut codes_index, + key: &str, + value: &str, +) -> Result<(), CodesError> { + let key = CString::new(key).unwrap(); + let value = CString::new(value).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_select_string(index, key.as_ptr(), value.as_ptr()); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_handle_new_from_index( + index: *mut codes_index, +) -> Result<*mut codes_handle, CodesError> { + let mut error_code: i32 = 0; + + let _g = CODES_LOCK.lock().unwrap(); + let codes_handle = eccodes_sys::codes_handle_new_from_index(index, &mut error_code); + + // special case! codes_handle_new_from_index returns -43 when there are no messages left in the index + // this is also indicated by a null pointer, which is handled upstream + if error_code == -43 { + return Ok(codes_handle); + } + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(codes_handle) +} diff --git a/src/intermediate_bindings.rs b/src/intermediate_bindings/mod.rs similarity index 99% rename from src/intermediate_bindings.rs rename to src/intermediate_bindings/mod.rs index 0397199..cf7f521 100644 --- a/src/intermediate_bindings.rs +++ b/src/intermediate_bindings/mod.rs @@ -7,6 +7,9 @@ //!to make ecCodes usage safer and easier, //!but they are unsafe as they operate on raw `codes_handle`. +#[cfg(feature = "experimental_index")] +pub mod codes_index; + use std::{ ffi::{CStr, CString}, ptr::{self, addr_of_mut}, @@ -68,6 +71,10 @@ pub unsafe fn codes_handle_new_from_file( } pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { + if handle.is_null() { + return Ok(()); + } + let error_code = eccodes_sys::codes_handle_delete(handle); if error_code != 0 { diff --git a/src/lib.rs b/src/lib.rs index 0ae82a3..08348f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,9 +25,9 @@ //!### Accessing GRIB files //! //!This crate provides an access to GRIB file by creating a -//![`CodesHandle`](codes_handle::CodesHandle) and reading messages from the file with it. +//![`CodesHandle`] and reading messages from the file with it. //! -//!The [`CodesHandle`](codes_handle::CodesHandle) can be constructed in two ways: +//!The [`CodesHandle`] can be constructed in two ways: //! //!- The main option is to use [`new_from_file()`](codes_handle::CodesHandle::new_from_file) function //!to open a file under provided [`path`](`std::path::Path`) with filesystem, @@ -41,8 +41,8 @@ //!Data (messages) inside the GRIB file can be accessed using the [`FallibleIterator`](`codes_handle::CodesHandle#impl-FallibleIterator`) //!by iterating over the `CodesHandle`. //! -//!The `FallibleIterator` returns a [`KeyedMessage`](codes_handle::KeyedMessage) structure which implements some -//!methods to access data values. The data inside `KeyedMessage` is provided directly as [`Key`](codes_handle::Key) +//!The `FallibleIterator` returns a [`KeyedMessage` structure which implements some +//!methods to access data values. The data inside `KeyedMessage` is provided directly as [`Key`] //!or as more specific data type. //! //!#### Example @@ -217,11 +217,18 @@ //! pub mod codes_handle; +#[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +pub mod codes_index; pub mod errors; mod intermediate_bindings; pub use codes_handle::{ CodesHandle, Key, KeyType, KeyedMessage, KeysIteratorFlags, NearestGridpoint, ProductKind, }; +#[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +pub use codes_index::CodesIndex; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; +pub use fallible_streaming_iterator::FallibleStreamingIterator; diff --git a/tests/handle.rs b/tests/handle.rs new file mode 100644 index 0000000..299a10b --- /dev/null +++ b/tests/handle.rs @@ -0,0 +1,48 @@ +use std::{path::Path, thread}; + +use eccodes::{CodesHandle, KeyType, ProductKind, FallibleStreamingIterator}; + +#[test] +fn thread_safety() { + thread::spawn(|| loop { + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB).unwrap(); + let current_message = handle.next().unwrap().unwrap(); + + for _ in 0..100 { + let _ = current_message.read_key("name").unwrap(); + + let str_key = current_message.read_key("name").unwrap(); + + match str_key.value { + KeyType::Str(_) => {} + _ => panic!("Incorrect variant of string key"), + } + + assert_eq!(str_key.name, "name"); + } + + drop(handle); + }); + + for _ in 0..1000 { + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB).unwrap(); + let current_message = handle.next().unwrap().unwrap(); + + let long_key = current_message + .read_key("numberOfPointsAlongAParallel") + .unwrap(); + + match long_key.value { + KeyType::Int(_) => {} + _ => panic!("Incorrect variant of long key"), + } + + assert_eq!(long_key.name, "numberOfPointsAlongAParallel"); + + drop(handle); + } +} diff --git a/tests/index.rs b/tests/index.rs new file mode 100644 index 0000000..5f62b26 --- /dev/null +++ b/tests/index.rs @@ -0,0 +1,262 @@ +#![cfg(feature = "experimental_index")] + +use std::{path::Path, thread}; + +use anyhow::Result; +use eccodes::{ + codes_index::Select, CodesError, CodesHandle, CodesIndex, FallibleStreamingIterator, KeyType, + ProductKind, +}; +use rand::Rng; + +#[test] +fn iterate_handle_from_index() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + let handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + + let counter = handle.count().unwrap(); + + assert_eq!(counter, 1); +} + +#[test] +fn read_index_messages() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + let mut handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + let current_message = handle.next().unwrap().unwrap(); + + { + let short_name = current_message.read_key("shortName").unwrap(); + match short_name.value { + KeyType::Str(val) => assert!(val == "2t"), + _ => panic!("Unexpected key type"), + }; + } + { + let level = current_message.read_key("level").unwrap(); + match level.value { + KeyType::Int(val) => assert!(val == 0), + _ => panic!("Unexpected key type"), + }; + } +} + +#[test] +fn collect_index_iterator() -> Result<()> { + let keys = vec!["typeOfLevel", "level"]; + let index = CodesIndex::new_from_keys(&keys)?; + let grib_path = Path::new("./data/iceland-levels.grib"); + + let index = index + .add_grib_file(grib_path)? + .select("typeOfLevel", "isobaricInhPa")? + .select("level", 700)?; + + let mut handle = CodesHandle::new_from_index(index, ProductKind::GRIB)?; + + let mut levels = vec![]; + + while let Some(msg) = handle.next()? { + levels.push(msg.clone()); + } + + assert_eq!(levels.len(), 5); + + Ok(()) +} + +#[test] +fn add_file_error() { + thread::spawn(|| { + let grib_path = Path::new("./data/iceland-levels.grib"); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let mut index_op = CodesIndex::new_from_keys(&keys).unwrap(); + + loop { + index_op = index_op.add_grib_file(grib_path).unwrap(); + } + }); + + thread::sleep(std::time::Duration::from_millis(250)); + + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let wrong_path = Path::new("./data/xxx.grib"); + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(wrong_path); + + assert!(index.is_err()); +} + +#[test] +fn index_panic() { + thread::spawn(|| { + let grib_path = Path::new("./data/iceland-levels.grib"); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let mut index_op = CodesIndex::new_from_keys(&keys).unwrap(); + + loop { + index_op = index_op.add_grib_file(grib_path).unwrap(); + } + }); + + thread::sleep(std::time::Duration::from_millis(250)); + + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let wrong_path = Path::new("./data/xxx.grib"); + let index = CodesIndex::new_from_keys(&keys).unwrap(); + + let result = std::panic::catch_unwind(|| index.add_grib_file(wrong_path).unwrap()); + + assert!(result.is_err()); +} + +#[test] +fn add_file_while_index_open() { + thread::spawn(|| { + let file_path = Path::new("./data/iceland-surface.idx"); + let mut index_op = CodesIndex::read_from_file(file_path).unwrap(); + + loop { + index_op = index_op + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + } + }); + + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let grib_path = Path::new("./data/iceland-surface.grib"); + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(grib_path); + + assert!(index.is_ok()); +} + +#[test] +fn add_file_to_read_index() { + let file_path = Path::new("./data/iceland-surface.idx"); + let grib_path = Path::new("./data/iceland-surface.grib"); + + let _index = CodesIndex::read_from_file(file_path) + .unwrap() + .add_grib_file(grib_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); +} + +#[test] +fn simulatenous_index_destructors() -> Result<()> { + let h1 = thread::spawn(|| -> anyhow::Result<(), CodesError> { + let mut rng = rand::thread_rng(); + let file_path = Path::new("./data/iceland-surface.idx"); + + for _ in 0..10 { + let sleep_time = rng.gen_range(1..30); // randomizing sleep time to hopefully catch segfaults + + let index_op = CodesIndex::read_from_file(file_path)? + .select("shortName", "2t")? + .select("typeOfLevel", "surface")? + .select("level", 0)? + .select("stepType", "instant")?; + + thread::sleep(std::time::Duration::from_millis(sleep_time)); + drop(index_op); + } + + Ok(()) + }); + + let h2 = thread::spawn(|| -> anyhow::Result<(), CodesError> { + let mut rng = rand::thread_rng(); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let grib_path = Path::new("./data/iceland-surface.grib"); + + for _ in 0..10 { + let sleep_time = rng.gen_range(1..42); // randomizing sleep time to hopefully catch segfaults + + let index = CodesIndex::new_from_keys(&keys)? + .add_grib_file(grib_path)? + .select("shortName", "2t")? + .select("typeOfLevel", "surface")? + .select("level", 0)? + .select("stepType", "instant")?; + + thread::sleep(std::time::Duration::from_millis(sleep_time)); + drop(index); + } + + Ok(()) + }); + + h1.join().unwrap()?; + h2.join().unwrap()?; + + Ok(()) +} + +#[test] +fn index_handle_interference() { + thread::spawn(|| { + let file_path = Path::new("./data/iceland.grib"); + + loop { + let handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB); + + assert!(handle.is_ok()); + } + }); + + let mut rng = rand::thread_rng(); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let grib_path = Path::new("./data/iceland.grib"); + + for _ in 0..10 { + let sleep_time = rng.gen_range(1..42); // randomizing sleep time to hopefully catch segfaults + + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(grib_path) + .unwrap(); + let i_handle = CodesHandle::new_from_index(index, ProductKind::GRIB); + + assert!(i_handle.is_ok()); + + thread::sleep(std::time::Duration::from_millis(sleep_time)); + } +}