Squashed commit of the following:

commit 58d92ac Author: Quba1 <[email protected]> Date: Sat Feb 3 17:07:40 2024 +0100 make index feature experimental commit 7275212 Author: Quba1 <[email protected]> Date: Sat Feb 3 16:18:58 2024 +0100 implement streaming iter and adapt tests disables lib tests for now also adds result to some tests commit 2512067 Author: Quba1 <[email protected]> Date: Fri Feb 2 19:56:05 2024 +0100 initial preparation commit 58ab75e Author: Quba1 <[email protected]> Date: Sat Feb 3 16:24:50 2024 +0100 safeguard all index functions commit 166d701 Author: Quba1 <[email protected]> Date: Sat Feb 3 12:53:37 2024 +0100 make ec_index default feature commit d3682ff Author: Quba1 <[email protected]> Date: Tue Jan 30 18:39:01 2024 +0100 clippy commit cff3157 Author: Quba1 <[email protected]> Date: Tue Jan 30 18:26:42 2024 +0100 add index handle interference test commit c099d58 Author: Quba1 <[email protected]> Date: Tue Jan 30 14:17:03 2024 +0100 add features to gh actions commit 9d8d83a Author: Quba1 <[email protected]> Date: Tue Jan 30 14:14:01 2024 +0100 add error and panic tests for codes_index commit bb2558c Author: Quba1 <[email protected]> Date: Tue Jan 30 13:10:02 2024 +0100 add mutex for problematic functions commit 9f8a137 Author: Quba1 <[email protected]> Date: Mon Jan 29 20:49:37 2024 +0100 add integration tests with multithreading commit 2590cb2 Author: Quba1 <[email protected]> Date: Mon Jan 29 08:14:51 2024 +0100 move high-level index tests to integration tests commit 952a111 Author: Quba1 <[email protected]> Date: Sun Jan 28 18:09:32 2024 +0100 actually, the API makes sense (and tests now pass) commit 5a30252 Author: Quba1 <[email protected]> Date: Sun Jan 28 18:05:35 2024 +0100 compiles but does not allow for editing index so misses the whole point of index commit 640122f Author: Quba1 <[email protected]> Date: Sun Jan 28 17:29:33 2024 +0100 WIP codes_handle iterator with index works commit 8909556 Author: Quba1 <[email protected]> Date: Sun Jan 28 16:41:39 2024 +0100 WIP mostly finish CodesIndex API and conversion to CH commit db3555c Author: Quba1 <[email protected]> Date: Sun Jan 28 13:02:08 2024 +0100 (deosn't compile) extract index binds to separate file commit 993efb9 Author: Quba1 <[email protected]> Date: Sat Jan 27 12:34:56 2024 +0100 fix testing in gh actions commit 95c59b6 Author: Quba1 <[email protected]> Date: Sat Jan 27 12:17:40 2024 +0100 update gh actions commit 4b0847e Author: Quba1 <[email protected]> Date: Sat Jan 27 12:16:20 2024 +0100 reolve cargo doc warnings commit 0e1ff8e Author: Quba1 <[email protected]> Date: Sat Jan 27 12:14:07 2024 +0100 mark featured modules in docs commit e76f3bc Author: Quba1 <[email protected]> Date: Sat Jan 27 12:04:01 2024 +0100 make codes_index a feature commit a3c25cb Author: Quba1 <[email protected]> Date: Sat Jan 27 11:51:41 2024 +0100 re-export CodesIndex in lib.rs for consistency commit 5df6d99 Author: Quba1 <[email protected]> Date: Wed May 17 14:55:57 2023 +0200 make index_handle public only within crate commit 31c7cde Merge: f6b3a2b f56a6f2 Author: Jakub Lewandowski <[email protected]> Date: Sat Jan 27 11:47:53 2024 +0100 Merge pull request #2 from asura6/initial-index-support initial index-support commit f56a6f2 Merge: 5b7f86d f6b3a2b Author: Jakub Lewandowski <[email protected]> Date: Sat Jan 27 11:47:13 2024 +0100 Merge branch 'codes-index' into initial-index-support commit 5b7f86d Author: Robin <[email protected]> Date: Tue May 9 19:16:56 2023 +0200 initial index-support
ScaleWeather · Feb 3, 2024 · a83238b · a83238b
1 parent f6b3a2b
commit a83238b
Show file tree

Hide file tree

Showing 17 changed files with 1,031 additions and 196 deletions.
diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml
@@ -23,13 +23,13 @@ jobs:
         cargo clean
     - name: Test with cargo
       run: |
-        RUST_BACKTRACE=full cargo test
+        RUST_BACKTRACE=full cargo test --features "experimental_index"
     - name: Check with clippy
       run: |
-        cargo clippy -- -W clippy::pedantic
+        cargo clippy --features "experimental_index"
     - name: Build release
       run: |
-        cargo build --release
+        cargo build --release --features "experimental_index"
 
   build-macos:
 
@@ -45,10 +45,10 @@ jobs:
         cargo clean
     - name: Test with cargo
       run: |
-        RUST_BACKTRACE=full cargo test
+        RUST_BACKTRACE=full cargo test --features "experimental_index"
     - name: Check with clippy
       run: |
-        cargo clippy -- -W clippy::pedantic
+        cargo clippy --features "experimental_index"
     - name: Build release
       run: |
-        cargo build --release
+        cargo build --release --features "experimental_index"
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -27,11 +27,11 @@ jobs:
         cargo clean
     - name: Build with cargo
       run: |
-        cargo build --release
+        cargo build --release --features "experimental_index"
         cargo clean
     - name: Test with cargo
       run: |
-        cargo test
+        cargo test --features "experimental_index"
         cargo clean
     - name: Benchmark with criterion
       run: |
@@ -53,11 +53,11 @@ jobs:
         cargo clean
     - name: Build with cargo
       run: |
-        cargo build --release
+        cargo build --release --features "experimental_index"
         cargo clean
     - name: Test with cargo
       run: |
-        cargo test
+        cargo test --features "experimental_index"
         cargo clean
     - name: Benchmark with criterion
       run: |

diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 name = "eccodes"
 description = "Unofficial high-level Rust bindings of the latest ecCodes release"
 repository = "https://github.com/ScaleWeather/eccodes"
-version = "0.8.0"
+version = "0.9.0"
 readme = "README.md"
 authors = ["Jakub Lewandowski <[email protected]>"]
 keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"]
@@ -18,7 +18,7 @@ edition = "2021"
 exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"]
 
 [dependencies]
-eccodes-sys = "0.5.1"
+eccodes-sys = "0.5.2"
 libc = "0.2"
 thiserror = "1.0"
 bytes = "1.5"
@@ -27,20 +27,26 @@ errno = "0.3"
 num-derive = "0.4.1"
 num-traits = "0.2"
 fallible-iterator = "0.3"
+fallible-streaming-iterator = "0.1.9"
 
 [dev-dependencies]
-eccodes-sys = "0.5.1"
 reqwest = { version = "0.11", features = ["rustls-tls"] }
 tokio = { version = "1.35", features = ["macros", "rt"] }
 criterion = "0.5"
 testing_logger = "0.1"
+rand = "0.8"
+anyhow = "1.0"
 
 [features]
 docs = ["eccodes-sys/docs"]
+experimental_index = []
 
 [package.metadata.docs.rs]
-features = ["docs"]
+features = ["docs", "experimental_index"]
 
 [[bench]]
 name = "main"
 harness = false
+
+[lib]
+doctest = false
diff --git a/benches/main.rs b/benches/main.rs
@@ -1,4 +1,4 @@
-use eccodes::FallibleIterator;
+use eccodes::FallibleStreamingIterator;
 use std::path::Path;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};

diff --git a/data/iceland-surface.idx b/data/iceland-surface.idx
diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs
@@ -1,14 +1,16 @@
-use eccodes_sys::codes_handle;
-use fallible_iterator::FallibleIterator;
+use std::ptr;
+
+use fallible_streaming_iterator::FallibleStreamingIterator;
 
 use crate::{
     codes_handle::{CodesHandle, KeyedMessage},
     errors::CodesError,
-    intermediate_bindings::{
-        codes_get_message_copy, codes_handle_delete, codes_handle_new_from_file,
-        codes_handle_new_from_message_copy,
-    },
+    intermediate_bindings::{codes_handle_delete, codes_handle_new_from_file},
 };
+#[cfg(feature = "experimental_index")]
+use crate::{intermediate_bindings::codes_index::codes_handle_new_from_index, CodesIndex};
+
+use super::GribFile;
 
 ///`FallibleIterator` implementation for `CodesHandle` to access GRIB messages inside file.
 ///
@@ -18,7 +20,7 @@ use crate::{
 ///Therefore this crate utilizes the `Iterator` to provide the access to GRIB messages in
 ///a safe and convienient way.
 ///
-///[`FallibleIterator`](fallible_iterator::FallibleIterator) is used instead of classic `Iterator`
+///[`FallibleIterator`] is used instead of classic `Iterator`
 ///because internal ecCodes functions can return error codes when the GRIB file
 ///is corrupted and for some other reasons. The usage of `FallibleIterator` is sligthly different
 ///than usage of `Iterator`, check its documentation for more details.
@@ -77,81 +79,141 @@ use crate::{
 ///## Errors
 ///The `next()` method will return [`CodesInternal`](crate::errors::CodesInternal)
 ///when internal ecCodes function returns non-zero code.
-impl FallibleIterator for CodesHandle {
+impl FallibleStreamingIterator for CodesHandle<GribFile> {
     type Item = KeyedMessage;
 
     type Error = CodesError;
 
-    fn next(&mut self) -> Result<Option<Self::Item>, Self::Error> {
-        let file_handle;
+    fn advance(&mut self) -> Result<(), Self::Error> {
         unsafe {
-            codes_handle_delete(self.file_handle)?;
-            file_handle = codes_handle_new_from_file(self.file_pointer, self.product_kind);
+            codes_handle_delete(self.unsafe_message.message_handle)?;
         }
 
-        match file_handle {
-            Ok(h) => {
-                self.file_handle = h;
+        // nullify message handle so that destructor is harmless
+        // it might be excessive but it follows the correct pattern
+        self.unsafe_message.message_handle = ptr::null_mut();
 
-                if self.file_handle.is_null() {
-                    Ok(None)
-                } else {
-                    let message = get_message_from_handle(h);
-                    Ok(Some(message))
-                }
-            }
-            Err(e) => Err(e),
+        let new_eccodes_handle =
+            unsafe { codes_handle_new_from_file(self.source.pointer, self.product_kind)? };
+
+        self.unsafe_message = KeyedMessage {
+            message_handle: new_eccodes_handle,
+            iterator_flags: None,
+            iterator_namespace: None,
+            keys_iterator: None,
+            keys_iterator_next_item_exists: false,
+            nearest_handle: None,
+        };
+
+        Ok(())
+    }
+
+    fn get(&self) -> Option<&Self::Item> {
+        if self.unsafe_message.message_handle.is_null() {
+            None
+        } else {
+            Some(&self.unsafe_message)
         }
     }
 }
 
-fn get_message_from_handle(handle: *mut codes_handle) -> KeyedMessage {
-    let new_handle;
-    let new_buffer;
+#[cfg(feature = "experimental_index")]
+impl FallibleStreamingIterator for CodesHandle<CodesIndex> {
+    type Item = KeyedMessage;
 
-    unsafe {
-        new_buffer = codes_get_message_copy(handle).expect(
-            "Getting message clone failed.
-        Please report this panic on Github",
-        );
-        new_handle = codes_handle_new_from_message_copy(&new_buffer);
+    type Error = CodesError;
+
+    fn advance(&mut self) -> Result<(), Self::Error> {
+        unsafe {
+            codes_handle_delete(self.unsafe_message.message_handle)?;
+        }
+
+        // nullify message handle so that destructor is harmless
+        // it might be excessive but it follows the correct pattern
+        self.unsafe_message.message_handle = ptr::null_mut();
+
+        let new_eccodes_handle = unsafe { codes_handle_new_from_index(self.source.pointer)? };
+
+        self.unsafe_message = KeyedMessage {
+            message_handle: new_eccodes_handle,
+            iterator_flags: None,
+            iterator_namespace: None,
+            keys_iterator: None,
+            keys_iterator_next_item_exists: false,
+            nearest_handle: None,
+        };
+
+        Ok(())
     }
 
-    KeyedMessage {
-        message_handle: new_handle,
-        iterator_flags: None,
-        iterator_namespace: None,
-        keys_iterator: None,
-        keys_iterator_next_item_exists: false,
-        nearest_handle: None,
+    fn get(&self) -> Option<&Self::Item> {
+        if self.unsafe_message.message_handle.is_null() {
+            None
+        } else {
+            Some(&self.unsafe_message)
+        }
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::codes_handle::{CodesHandle, KeyType, KeyedMessage, ProductKind};
-    use crate::FallibleIterator;
+    use crate::codes_handle::{CodesHandle, KeyType, ProductKind};
+    use anyhow::Result;
+    use fallible_streaming_iterator::FallibleStreamingIterator;
     use std::path::Path;
 
     #[test]
-    fn iterator_fn() {
+    fn iterator_lifetimes() -> Result<()> {
+        let file_path = Path::new("./data/iceland-levels.grib");
+        let product_kind = ProductKind::GRIB;
+        let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap();
+
+        let msg1 = handle.next()?.unwrap();
+        let key1 = msg1.read_key("typeOfLevel")?;
+
+        let msg2 = handle.next()?.unwrap();
+        let key2 = msg2.read_key("typeOfLevel")?;
+
+        let msg3 = handle.next()?.unwrap();
+        let key3 = msg3.read_key("typeOfLevel")?;
+
+        assert_eq!(key1.value, KeyType::Str("isobaricInhPa".to_string()));
+        assert_eq!(key2.value, KeyType::Str("isobaricInhPa".to_string()));
+        assert_eq!(key3.value, KeyType::Str("isobaricInhPa".to_string()));
+
+        Ok(())
+    }
+
+    #[test]
+    fn iterator_fn() -> Result<()> {
         let file_path = Path::new("./data/iceland-surface.grib");
         let product_kind = ProductKind::GRIB;
 
         let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap();
 
-        while let Some(msg) = handle.next().unwrap() {
-            let key = msg.read_key("shortName").unwrap();
+        while let Some(msg) = handle.next()? {
+            let key = msg.read_key("shortName")?;
 
             match key.value {
                 KeyType::Str(_) => {}
                 _ => panic!("Incorrect variant of string key"),
             }
         }
 
-        let handle = CodesHandle::new_from_file(file_path, product_kind).unwrap();
+        Ok(())
+    }
 
-        let handle_collected: Vec<KeyedMessage> = handle.collect().unwrap();
+    #[test]
+    fn iterator_collected() -> Result<()> {
+        let file_path = Path::new("./data/iceland-surface.grib");
+        let product_kind = ProductKind::GRIB;
+        let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap();
+
+        let mut handle_collected = vec![];
+
+        while let Some(msg) = handle.next()? {
+            handle_collected.push(msg.clone());
+        }
 
         for msg in handle_collected {
             let key = msg.read_key("name").unwrap();
@@ -160,6 +222,8 @@ mod tests {
                 _ => panic!("Incorrect variant of string key"),
             }
         }
+
+        Ok(())
     }
 
     #[test]
@@ -178,24 +242,23 @@ mod tests {
     }
 
     #[test]
-    fn iterator_collect() {
+    fn iterator_filter() -> Result<()> {
         let file_path = Path::new("./data/iceland.grib");
         let product_kind = ProductKind::GRIB;
 
-        let handle = CodesHandle::new_from_file(file_path, product_kind).unwrap();
+        let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap();
 
         // Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface"
         // First, filter and collect the messages to get those that we want
-        let mut level: Vec<KeyedMessage> = handle
-            .filter(|msg| {
-                Ok(
-                    msg.read_key("shortName")?.value == KeyType::Str("msl".to_string())
-                        && msg.read_key("typeOfLevel")?.value
-                            == KeyType::Str("surface".to_string()),
-                )
-            })
-            .collect()
-            .unwrap();
+        let mut level = vec![];
+
+        while let Some(msg) = handle.next()? {
+            if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string())
+                && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string())
+            {
+                level.push(msg.clone());
+            }
+        }
 
         // Now unwrap and access the first and only element of resulting vector
         // Find nearest modifies internal KeyedMessage fields so we need mutable reference
@@ -211,5 +274,7 @@ mod tests {
             "value: {}, distance: {}",
             nearest_gridpoints[3].value, nearest_gridpoints[3].distance
         );
+
+        Ok(())
     }
 }