From eef741db5d4f06c4b961fadae08bef31d6c440ba Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Tue, 6 Feb 2024 21:40:46 +0100 Subject: [PATCH] WIP major docs overhaul --- Cargo.toml | 3 +- README.md | 162 ++++++++--------- src/codes_handle/iterator.rs | 7 +- src/codes_handle/mod.rs | 20 ++- src/codes_index.rs | 6 +- src/codes_nearest.rs | 3 + src/errors.rs | 28 +-- src/keyed_message/mod.rs | 26 +-- src/keyed_message/write.rs | 8 +- src/keys_iterator.rs | 2 + src/lib.rs | 330 +++++++++++++++++++---------------- src/message_ndarray.rs | 30 +++- tests/index.rs | 2 +- 13 files changed, 357 insertions(+), 270 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0659a4f..31493df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" -version = "0.9.0" +version = "0.10.0" readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] @@ -16,6 +16,7 @@ categories = [ license = "Apache-2.0" edition = "2021" exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] +rust-version = "1.70.0" [dependencies] eccodes-sys = { version = "0.5.2", default-features = false } diff --git a/README.md b/README.md index 58273ff..75b623a 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,29 @@ # eccodes -[![License](https://img.shields.io/github/license/ScaleWeather/eccodes)](https://choosealicense.com/licenses/apache-2.0/) -[![Crates.io](https://img.shields.io/crates/v/eccodes)](https://crates.io/crates/eccodes) -[![dependency status](https://deps.rs/repo/github/ScaleWeather/eccodes/status.svg)](https://deps.rs/repo/github/ScaleWeather/eccodes) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/ScaleWeather/eccodes/rust.yml?branch=main&label=cargo%20build)](https://github.com/ScaleWeather/eccodes/actions) -[![docs.rs](https://img.shields.io/docsrs/eccodes)](https://docs.rs/eccodes) - -This crate contains safe high-level bindings for ecCodes library. +[![Github Repository](https://img.shields.io/badge/Github-Repository-blue?style=flat-square&logo=github&color=blue)](https://github.com/ScaleWeather/eccodes) +[![Crates.io](https://img.shields.io/crates/v/eccodes?style=flat-square)](https://crates.io/crates/eccodes) +[![License](https://img.shields.io/github/license/ScaleWeather/eccodes?style=flat-square)](https://choosealicense.com/licenses/apache-2.0/) +[![dependency status](https://deps.rs/repo/github/ScaleWeather/eccodes/status.svg?style=flat-square)](https://deps.rs/repo/github/ScaleWeather/eccodes) +![Crates.io MSRV](https://img.shields.io/crates/msrv/eccodes?style=flat-square) +![ecCodes version](https://img.shields.io/badge/ecCodes-%E2%89%A52.24.0-blue?style=flat-square&color=blue) + +This crate contains (mostly) safe high-level bindings for ecCodes library. Bindings can be considered safe mainly because all crate structures -take ownership of the data in memory before passing the raw pointer to ecCodes. +will take ownership of the data in memory before passing the raw pointer to ecCodes. **Currently only reading of GRIB files is supported.** -As the API of this crate differs significantly from the API of ecCodes library -make sure to read its [documentation](https://docs.rs/eccodes). -Read [this section](#crate-safety) to learn more about design decisions of this crate. +Because of the ecCodes library API characteristics theses bindings are +rather thick wrapper to make this crate safe and convenient to use. -**If you want to see more features released quicker do not hesitate to contribute and check out Github repository.** All submitted issues and pull requests are welcome. +This crate officially supports mainly Linux platforms same as the ecCodes library. +But it is possible to install ecCodes on MacOS and this crate successfully compiles and all tests pass. -[ecCodes](https://confluence.ecmwf.int/display/ECC/ecCodes+Home) is an -open-source library for reading and writing GRIB and BUFR files -developed by [European Centre for Medium-Range Weather Forecasts](https://www.ecmwf.int/). +**If you want to see more features released quicker do not hesitate +to contribute and check out [Github repository](https://github.com/ScaleWeather/eccodes).** -This crate officially supports mainly Linux platforms as the ecCodes library supports them. -But it is possible to install ecCodes on MacOS and this crate successfully compiles and all tests pass. +[ecCodes](https://confluence.ecmwf.int/display/ECC/ecCodes+Home) is an open-source library +for reading and writing GRIB and BUFR files developed by [European Centre for Medium-Range Weather Forecasts](https://www.ecmwf.int/). ## Usage @@ -58,63 +58,55 @@ export PKG_CONFIG_PATH=/lib/pkgconfig export LD_LIBRARY_PATH=/lib ``` -### Accessing GRIB files - -This crate provides an access to GRIB file by creating a -`CodesHandle` and reading with it messages from the file. +### Working with GRIB files -The `CodesHandle` can be constructed in two ways: +To access a GRIB file you need to create `CodesHandle` with one of provided constructors. -- The main option is to use `new_from_file()` function -to open a file under provided `path` with filesystem, -when copying whole file into memory is not desired or not necessary. +GRIB files consist of messages which represent data fields at specific time and level. +Messages are represented by the `KeyedMessage` structure. -- Alternatively `new_from_memory()` function can be used -to access a file that is already in memory. For example, when file is downloaded from the internet -and does not need to be saved on hard drive. -The file must be stored in `bytes::Bytes`. +`CodesHandle` implements `FallibleStreamingIterator` +which allows you to iterate over messages in the file. The iterator returns `&KeyedMessage` which valid until next iteration. +`KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. +You can also use `try_clone()` to clone the message and prolong its lifetime. -Data (messages) inside the GRIB file can be accessed using the `FallibleIterator` -by iterating over the `CodesHandle`. +Data defining and contained by `KeyedMessage` is represented by `Key`s. +You can read them directly with `read_key()`, use `KeysIterator` +to iterate over them or use `CodesNearest` to get the values of four nearest gridpoints for given coordinates. -The `FallibleIterator` returns a `KeyedMessage` structure which implements some -methods to access data values. The data inside `KeyedMessage` is provided directly as `Key` -or as more specific data type. +You can also modify the message with `write_key()` and write it to a new file with `write_to_file()`. #### Example ```rust // We are reading the mean sea level pressure for 4 gridpoints -// nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC +// nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC // from ERA5 Climate Reanalysis +use eccodes::{ProductKind, CodesHandle, KeyType}; +use eccodes::FallibleStreamingIterator; + // Open the GRIB file and create the CodesHandle let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - -let handle = CodesHandle::new_from_file(file_path, product_kind)?; - -// Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface" -// First, filter and collect the messages to get those that we want -let mut level: Vec = handle - .filter(|msg| { - - Ok(msg.read_key("shortName")?.value == Str("msl".to_string()) - && msg.read_key("typeOfLevel")?.value == Str("surface".to_string())) - }) - .collect()?; - -// Now unwrap and access the first and only element of resulting vector -// Find nearest modifies internal KeyedMessage fields so we need mutable reference -let level = &mut level[0]; - -// Get the four nearest gridpoints of Reykjavik -let nearest_gridpoints = level.find_nearest(64.13, -21.89)?; - -// Print value and distance of the nearest gridpoint -println!("value: {}, distance: {}", - nearest_gridpoints[3].value, - nearest_gridpoints[3].distance); +let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + +// Use iterator to find a message with shortName "msl" and typeOfLevel "surface" +// We can use while let or for_each() to iterate over the messages +while let Some(msg) = handle.next()? { + if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) + && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) { + + // Create CodesNearest for given message + let nearest_gridpoints = msg.codes_nearest()? + // Find the nearest gridpoints to Reykjavik + .find_nearest(64.13, -21.89)?; + // Print value and distance of the nearest gridpoint + println!("value: {}, distance: {}", + nearest_gridpoints[3].value, + nearest_gridpoints[3].distance); + } +} ``` ### Writing GRIB files @@ -127,34 +119,46 @@ modify the keys and write to new file. You can find a detailed example of setting keys and writing message to file in the documentation. -### Features +## Errors and panics -- `docs` - builds the create without linking ecCodes, particularly useful when building the documentation -on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +This crate aims to return error whenever possible, even if the error is caused by implementation bug. +As ecCodes is often used in scientific applications with long and extensive jobs, +this allows the user to handle the error in the way that suits them best and not risk crashes. -To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` +All error descriptions are provided in the `errors` module. +Destructors, which cannot panic, report errors through the `log` crate. -```toml -[package.metadata.docs.rs] -features = ["eccodes/docs"] -``` +None of the functions in this crate explicitly panics. +However, users should not that dependencies might panic in some edge cases. + +## Safety -## Crate safety +This crate aims to be as safe as possible and a lot of effort has been put into testing its safety. +Moreover, pointers are always checked for null before being dereferenced. -Because the ecCodes library API heavily relies on raw pointers simply making ecCodes functions callable without `unsafe` block would still allow for creation of dangling pointers and use-after-free, and the crate would not be truly safe. Therefore these bindings are rather thick wrapper as they need to take full ownership of accessed data to make the code safe. Having the data and pointers contained in dedicated data structures is also an occasion to make this crate API more convenient to use than the original ecCodes API (which is not really user-friendly). +That said, neither main developer nor contributors have expertise in unsafe Rust and bugs might have +slipped through. We are also not responsible for bugs in the ecCodes library. -## Roadmap +If you find a bug or have a suggestion, feel free to discuss it on Github. -_(Functions from ecCodes API wrapped at given stage are marked in parentheses)_ +## Features -- [x] Reading GRIB files - - [x] Creating CodesHandle from file and from memory (`codes_handle_new_from_file`, `codes_handle_delete`) - - [x] Iterating over GRIB messages with `Iterator` (`codes_get_message`, `codes_get_message_copy`, `codes_handle_new_from_message`, `codes_handle_new_from_message_copy`) - - [x] Reading keys from messages (`codes_get_double`, `codes_get_long`, `codes_get_string`, `codes_get_double_array`, `codes_get_long_array`, `codes_get_size`, `codes_get_length`, `codes_get_native_type`) - - [x] Iterating over key names with `Iterator` (`codes_grib_iterator_new`, `codes_grib_iterator_next`, `codes_keys_iterator_get_name`, `codes_keys_iterator_rewind`, `codes_grib_iterator_delete`) - - [x] Finding nearest data points for given coordinates (`codes_grib_nearest_new`, `codes_grib_nearest_find`, `codes_grib_nearest_delete`) -- [x] Writing GRIB files (`codes_set_double`, `codes_set_long`, `codes_set_string`, `codes_set_double_array`, `codes_set_long_array`,`codes_set_length`) -- [ ] Reading and writing BUFR files +- `message_ndarray` - enables support for converting `KeyedMessage` to `ndarray::Array`. +This feature is enabled by default. It is currently tested only with simple lat-lon grids. + +- `experimental_index` - enables support for creating and using index files for GRIB files. +This feature experimental and disabled by default. If you want to use it, please read +the information provided in [`codes_index`] documentation. + +- `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation +on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). + +To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` + +```text +[package.metadata.docs.rs] +features = ["eccodes/docs"] +``` ## License diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index 2902138..3350ea0 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -77,7 +77,7 @@ use super::GribFile; /// let mut handle_collected = vec![]; /// /// while let Some(msg) = handle.next()? { -/// handle_collected.push(msg.clone()); +/// handle_collected.push(msg.try_clone()?); /// } /// # Ok(()) /// # } @@ -123,6 +123,7 @@ impl FallibleStreamingIterator for CodesHandle { } #[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] impl FallibleStreamingIterator for CodesHandle { type Item = KeyedMessage; @@ -215,7 +216,7 @@ mod tests { let mut handle_collected = vec![]; while let Some(msg) = handle.next()? { - handle_collected.push(msg.clone()); + handle_collected.push(msg.try_clone()?); } for msg in handle_collected { @@ -278,7 +279,7 @@ mod tests { if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) { - level.push(msg.clone()); + level.push(msg.try_clone()?); } } diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 5702dfa..b36af97 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -1,5 +1,5 @@ -//!Main crate module containing definition of `CodesHandle` -//!and all associated functions and data structures +//! Definition and constructors of `CodesHandle` +//! used for accessing GRIB files #[cfg(feature = "experimental_index")] use crate::{codes_index::CodesIndex, intermediate_bindings::codes_index_delete}; @@ -20,14 +20,22 @@ use std::{ mod iterator; #[derive(Debug)] -#[doc(hidden)] pub struct GribFile { pointer: *mut FILE, } -///Main structure used to operate on the GRIB file. -///It takes a full ownership of the accessed file. -///It can be constructed either using a file or a memory buffer. +/// Main structure used to operate on the GRIB file, which takes a full ownership of the accessed file. +/// +/// It can be constructed either using a file or a memory buffer. +/// +/// - Use [`new_from_file()`](CodesHandle::new_from_file) +/// to open a file under provided [`path`](`std::path::Path`) using filesystem, +/// when copying whole file into memory is not desired or not necessary. +/// +/// - Alternatively use [`new_from_memory()`](CodesHandle::new_from_memory) +/// to access a file that is already in memory. For example, when file is downloaded from the internet +/// and does not need to be saved on hard drive. +/// The file must be stored in [`bytes::Bytes`](https://docs.rs/bytes/1.1.0/bytes/struct.Bytes.html). #[derive(Debug)] pub struct CodesHandle { _data: DataContainer, diff --git a/src/codes_index.rs b/src/codes_index.rs index b25ea07..949b7bd 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -1,5 +1,7 @@ -//!Main crate module containing definition of `CodesIndex` -//!and all associated functions and data structures +#![cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +//! ⚠️ **EXPERIMENTAL FEATURE - POSSIBLY UNSAFE** ⚠️ \ +//! Definition of `CodesIndex` and associated functions +//! used for efficient selection of messages from GRIB file use crate::{ codes_handle::SpecialDrop, diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 0024843..294cf3e 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -1,3 +1,6 @@ +//! Definition and associated functions of `CodesNearest` +//! used for finding nearest gridpoints in `KeyedMessage` + use std::ptr::null_mut; use eccodes_sys::codes_nearest; diff --git a/src/errors.rs b/src/errors.rs index 17abce1..31abdb4 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,29 +1,23 @@ -//!Module containing all error types used by the crate +//! Definition of errors returned by this crate //! -//!This crate uses [`thiserror`] crate to define its error types. +//! This crate uses [`thiserror`] crate to define its error types. //! -//!If you encounter an error that you believe is a result of implementation bug -//!rather then user's mistake post an issue on Github. +//! If you encounter an error that you believe is a result of implementation bug +//! rather then user's mistake post an issue on Github. use errno::Errno; use num_derive::FromPrimitive; use thiserror::Error; -#[derive(Error, Debug)] ///Errors returned by the crate's functions. ///These are the only errors that the user may face. +#[derive(Error, Debug)] pub enum CodesError { ///Returned when ecCodes library function returns an error code. ///Check [`CodesInternal`] for more details. #[error("ecCodes function returned a non-zero code {0}")] Internal(#[from] CodesInternal), - #[cfg(feature = "message_ndarray")] - /// Returned when function in `message_ndarray` module cannot convert - /// the message to ndarray. Check [`MessageNdarrayError`] for more details. - #[error("error occured while converting KeyedMessage to ndarray {0}")] - NdarrayConvert(#[from] MessageNdarrayError), - ///Returned when one of libc functions returns a non-zero error code. ///Check libc documentation for details of the errors. ///For libc reference check these websites: ([1](https://man7.org/linux/man-pages/index.html)) @@ -71,12 +65,18 @@ pub enum CodesError { /// it cannot be guaranteed that the null pointer is not caused by the user's mistake. #[error("Null pointer encountered where it should not be")] NullPtr, + + /// Returned when function in `message_ndarray` module cannot convert + /// the message to ndarray. Check [`MessageNdarrayError`] for more details. + #[cfg(feature = "message_ndarray")] + #[error("error occured while converting KeyedMessage to ndarray {0}")] + NdarrayConvert(#[from] MessageNdarrayError), } +/// Errors returned by the `message_ndarray` module. #[cfg(feature = "message_ndarray")] #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] -#[derive(Error, Debug)] -/// Errors returned by the `message_ndarray` module. +#[derive(PartialEq, Clone, Error, Debug)] pub enum MessageNdarrayError { /// Returned when functions converting to ndarray cannot correctly /// read key necessary for the conversion. @@ -99,9 +99,9 @@ pub enum MessageNdarrayError { IntCasting(#[from] std::num::TryFromIntError), } -#[derive(Copy, Eq, PartialEq, Clone, Ord, PartialOrd, Hash, Error, Debug, FromPrimitive)] ///Errors returned by internal ecCodes library functions. ///Copied directly from the ecCodes API. +#[derive(Copy, Eq, PartialEq, Clone, Ord, PartialOrd, Hash, Error, Debug, FromPrimitive)] pub enum CodesInternal { ///No error #[error("No error")] diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index 8b305cf..c0198ea 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -1,3 +1,6 @@ +//! Definition of `KeyedMessage` and its associated functions +//! used for reading and writing data of given variable from GRIB file + mod read; mod write; @@ -5,7 +8,7 @@ use eccodes_sys::codes_handle; use log::warn; use std::ptr::null_mut; -use crate::intermediate_bindings::{codes_handle_clone, codes_handle_delete}; +use crate::{intermediate_bindings::{codes_handle_clone, codes_handle_delete}, CodesError}; ///Structure used to access keys inside the GRIB file message. ///All data (including data values) contained by the file can only be accessed @@ -46,19 +49,18 @@ pub enum KeyType { Bytes(Vec), } -impl Clone for KeyedMessage { +impl KeyedMessage { ///Custom function to clone the `KeyedMessage`. This function comes with memory overhead. - ///During clone iterator flags and namespace are not copied, and the iterator is reset. /// - /// # Panics - /// This function will panic if ecCodes fails to clone the message. - fn clone(&self) -> KeyedMessage { + /// # Errors + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. + pub fn try_clone(&self) -> Result { let new_handle = - unsafe { codes_handle_clone(self.message_handle).expect("Cannot clone the message") }; + unsafe { codes_handle_clone(self.message_handle)? }; - KeyedMessage { + Ok(KeyedMessage { message_handle: new_handle, - } + }) } } @@ -102,7 +104,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle.next()?.context("Message not some")?; - let cloned_message = current_message.clone(); + let cloned_message = current_message.try_clone()?; assert_ne!( current_message.message_handle, @@ -118,7 +120,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg = handle.next()?.context("Message not some")?.clone(); + let msg = handle.next()?.context("Message not some")?.try_clone()?; let _ = handle.next()?; drop(handle); @@ -140,7 +142,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?.clone(); + let current_message = handle.next()?.context("Message not some")?.try_clone()?; let _kiter = current_message.default_keys_iterator()?; let _niter = current_message.codes_nearest()?; diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs index 7ee0c1f..62259e4 100644 --- a/src/keyed_message/write.rs +++ b/src/keyed_message/write.rs @@ -90,7 +90,7 @@ impl KeyedMessage { /// let file_path = Path::new("./data/iceland.grib"); /// /// let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - /// let mut current_message = handle.next()?.context("no message")?.clone(); + /// let mut current_message = handle.next()?.context("no message")?.try_clone()?; /// /// let new_key = Key { /// name: "centre".to_string(), @@ -164,7 +164,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?.clone(); + let current_message = handle.next()?.context("Message not some")?.try_clone()?; drop(handle); @@ -202,7 +202,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.next()?.context("Message not some")?.clone(); + let mut current_message = handle.next()?.context("Message not some")?.try_clone()?; let old_key = current_message.read_key("centre")?; @@ -227,7 +227,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.next()?.context("Message not some")?.clone(); + let mut current_message = handle.next()?.context("Message not some")?.try_clone()?; let old_key = current_message.read_key("centre")?; diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 6d6bf60..2329ab6 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -1,3 +1,5 @@ +//! Definition of `KeysIterator` used for iterating through keys in `KeyedMessage` + use eccodes_sys::codes_keys_iterator; use fallible_iterator::FallibleIterator; use log::warn; diff --git a/src/lib.rs b/src/lib.rs index 35236aa..e29f29a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,175 +1,211 @@ #![warn(clippy::pedantic)] #![allow(clippy::cast_possible_wrap)] -//!# Unofficial high-level safe Rust bindings to ecCodes library -//! -//!This crate contains safe high-level bindings for ecCodes library. -//!Bindings can be considered safe mainly because all crate structures -//!will take ownership of the data in memory before passing the raw pointer to ecCodes. -//!**Currently only reading of GRIB files is supported.** -//! -//!Because of the ecCodes library API characteristics theses bindings are -//!rather thick wrapper to make this crate safe and convenient to use. -//! -//!This crate officially supports mainly Linux platforms as the ecCodes library supports them. -//!But it is possible to install ecCodes on MacOS and this crate successfully compiles and all tests pass. -//! -//!If you want to see more features released quicker do not hesitate -//!to contribute and check out [Github repository](https://github.com/ScaleWeather/eccodes). -//! -//![ecCodes](https://confluence.ecmwf.int/display/ECC/ecCodes+Home) is an open-source library -//!for reading and writing GRIB and BUFR files developed by [European Centre for Medium-Range Weather Forecasts](https://www.ecmwf.int/). -//! -//!## Usage -//! -//!### Accessing GRIB files -//! -//!This crate provides an access to GRIB file by creating a -//![`CodesHandle`] and reading messages from the file with it. -//! -//!The [`CodesHandle`] can be constructed in two ways: -//! -//!- The main option is to use [`new_from_file()`](codes_handle::CodesHandle::new_from_file) function -//!to open a file under provided [`path`](`std::path::Path`) with filesystem, -//!when copying whole file into memory is not desired or not necessary. -//! -//!- Alternatively [`new_from_memory()`](codes_handle::CodesHandle::new_from_memory) function can be used -//!to access a file that is already in memory. For example, when file is downloaded from the internet -//!and does not need to be saved on hard drive. -//!The file must be stored in [`bytes::Bytes`](https://docs.rs/bytes/1.1.0/bytes/struct.Bytes.html). -//! -//!Data (messages) inside the GRIB file can be accessed using the [`FallibleIterator`](`codes_handle::CodesHandle#impl-FallibleIterator`) -//!by iterating over the `CodesHandle`. -//! -//!The `FallibleIterator` returns a [`KeyedMessage` structure which implements some -//!methods to access data values. The data inside `KeyedMessage` is provided directly as [`Key`] -//!or as more specific data type. -//! -//!#### Example -//! -//!``` -//!// We are reading the mean sea level pressure for 4 gridpoints -//!// nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC -//!// from ERA5 Climate Reanalysis -//! -//!// Open the GRIB file and create the CodesHandle -//! use eccodes::{ProductKind, CodesHandle, KeyType}; +//! # Unofficial high-level safe Rust bindings to ecCodes library +//! +//! [![Github Repository](https://img.shields.io/badge/Github-Repository-blue?style=flat-square&logo=github&color=blue)](https://github.com/ScaleWeather/eccodes) +//! [![Crates.io](https://img.shields.io/crates/v/eccodes?style=flat-square)](https://crates.io/crates/eccodes) +//! [![License](https://img.shields.io/github/license/ScaleWeather/eccodes?style=flat-square)](https://choosealicense.com/licenses/apache-2.0/) \ +//! [![dependency status](https://deps.rs/repo/github/ScaleWeather/eccodes/status.svg?style=flat-square)](https://deps.rs/repo/github/ScaleWeather/eccodes) +//! ![Crates.io MSRV](https://img.shields.io/crates/msrv/eccodes?style=flat-square) +//! ![ecCodes version](https://img.shields.io/badge/ecCodes-%E2%89%A52.24.0-blue?style=flat-square&color=blue) +//! +//! This crate contains (mostly) safe high-level bindings for ecCodes library. +//! Bindings can be considered safe mainly because all crate structures +//! will take ownership of the data in memory before passing the raw pointer to ecCodes. +//! +//! **Currently only reading of GRIB files is supported.** +//! +//! Because of the ecCodes library API characteristics theses bindings are +//! rather thick wrapper to make this crate safe and convenient to use. +//! +//! This crate officially supports mainly Linux platforms same as the ecCodes library. +//! But it is possible to install ecCodes on MacOS and this crate successfully compiles and all tests pass. +//! +//! If you want to see more features released quicker do not hesitate +//! to contribute and check out [Github repository](https://github.com/ScaleWeather/eccodes). +//! +//! [ecCodes](https://confluence.ecmwf.int/display/ECC/ecCodes+Home) is an open-source library +//! for reading and writing GRIB and BUFR files developed by [European Centre for Medium-Range Weather Forecasts](https://www.ecmwf.int/). +//! +//! ## Errors and panics +//! +//! This crate aims to return error whenever possible, even if the error is caused by implementation bug. +//! As ecCodes is often used in scientific applications with long and extensive jobs, +//! this allows the user to handle the error in the way that suits them best and not risk crashes. +//! +//! All error descriptions are provided in the [`errors`] module. +//! Destructors, which cannot panic, report errors through the `log` crate. +//! +//! None of the functions in this crate explicitly panics. +//! However, users should not that dependencies might panic in some edge cases. +//! +//! ## Safety +//! +//! This crate aims to be as safe as possible and a lot of effort has been put into testing its safety. +//! Moreover, pointers are always checked for null before being dereferenced. +//! +//! That said, neither main developer nor contributors have expertise in unsafe Rust and bugs might have +//! slipped through. We are also not responsible for bugs in the ecCodes library. +//! +//! If you find a bug or have a suggestion, feel free to discuss it on Github. +//! +//! ## Features +//! +//! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. +//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. +//! +//! - `experimental_index` - enables support for creating and using index files for GRIB files. +//! This feature experimental and disabled by default. If you want to use it, please read +//! the information provided in [`codes_index`] documentation. +//! +//! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation +//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! +//! To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` +//! +//! ```text +//! [package.metadata.docs.rs] +//! features = ["eccodes/docs"] +//! ``` +//! +//! ## Usage +//! +//! To access a GRIB file you need to create [`CodesHandle`] with one of provided constructors. +//! +//! GRIB files consist of messages which represent data fields at specific time and level. +//! Messages are represented by the [`KeyedMessage`] structure. +//! +//! [`CodesHandle`] implements [`FallibleStreamingIterator`](CodesHandle#impl-FallibleStreamingIterator-for-CodesHandle) +//! which allows you to iterate over messages in the file. The iterator returns `&KeyedMessage` which valid until next iteration. +//! `KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. +//! You can also use [`try_clone()`](KeyedMessage::try_clone) to clone the message and prolong its lifetime. +//! +//! Data defining and contained by `KeyedMessage` is represented by [`Key`]s. +//! You can read them directly with [`read_key()`](KeyedMessage::read_key), use [`KeysIterator`](KeyedMessage) +//! to iterate over them or use [`CodesNearest`] to get the values of four nearest gridpoints for given coordinates. +//! +//! You can also modify the message with [`write_key()`](KeyedMessage::write_key) and write +//! it to a new file with [`write_to_file()`](KeyedMessage::write_to_file). +//! +//! #### Example 1 - Reading GRIB file +//! +//! ``` +//! // We are reading the mean sea level pressure for 4 gridpoints +//! // nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC +//! // from ERA5 Climate Reanalysis +//! +//! use eccodes::{ProductKind, CodesHandle, KeyType}; //! # use std::path::Path; -//! use eccodes::FallibleStreamingIterator; +//! use eccodes::FallibleStreamingIterator; //! # //! # fn main() -> anyhow::Result<()> { +//! +//! // Open the GRIB file and create the CodesHandle //! let file_path = Path::new("./data/iceland.grib"); //! let product_kind = ProductKind::GRIB; -//! //! let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -//! -//! // Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface" -//! // First, filter and collect the messages to get those that we want +//! +//! // Use iterator to find a message with shortName "msl" and typeOfLevel "surface" +//! // We can use while let or for_each() to iterate over the messages //! while let Some(msg) = handle.next()? { //! if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) -//! && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) { +//! && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) { //! -//! // Get the four nearest gridpoints of Reykjavik -//! let nearest_gridpoints = msg.codes_nearest()?.find_nearest(64.13, -21.89)?; -//! -//! // Print value and distance of the nearest gridpoint -//! println!("value: {}, distance: {}", -//! nearest_gridpoints[3].value, -//! nearest_gridpoints[3].distance); +//! // Create CodesNearest for given message +//! let nearest_gridpoints = msg.codes_nearest()? +//! // Find the nearest gridpoints to Reykjavik +//! .find_nearest(64.13, -21.89)?; +//! +//! // Print value and distance of the nearest gridpoint +//! println!("value: {}, distance: {}", +//! nearest_gridpoints[3].value, +//! nearest_gridpoints[3].distance); //! } //! } //! # Ok(()) //! # } -//!``` -//! -//!### Writing GRIB files -//! -//!The crate provides a basic support for setting `KeyedMessage` keys -//!and writing GRIB files. The easiests (and safest) way to create a -//!new custom message is to copy exisitng one from other GRIB file, -//!modify the keys and write to new file. -//! -//!#### Example -//! -//!```rust -//! use eccodes::FallibleStreamingIterator; -//! use eccodes::{CodesHandle, Key, KeyType, ProductKind}; -//! # use std::{fs::remove_file, path::Path}; +//! ``` //! -//! # fn main() -> anyhow::Result<()> { -//! // We are computing the temperature at 850hPa as an average -//! // of 900hPa and 800hPa and writing it to a new file. -//! let file_path = Path::new("./data/iceland-levels.grib"); -//! let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; -//! -//! // We need a similar message to edit, -//! // in this case we can use temperature at 700hPa -//! let mut new_msg = vec![]; -//! -//! // Get temperatures at 800hPa and 900hPa -//! let mut t800 = vec![]; -//! let mut t900 = vec![]; -//! -//! while let Some(msg) = handle.next()? { -//! if msg.read_key("shortName")?.value == KeyType::Str("t".to_string()) { -//! if msg.read_key("level")?.value == KeyType::Int(700) { -//! new_msg.push(msg.clone()); -//! } +//! #### Example 2 - Writing GRIB files //! -//! if msg.read_key("level")?.value == KeyType::Int(800) { -//! if let KeyType::FloatArray(vals) = msg.read_key("values")?.value { -//! t800 = vals; -//! } -//! } +//! ```rust +//! // The crate provides basic support for setting `KeyedMessage` keys +//! // and writing GRIB files. The easiests (and safest) way to create a +//! // new custom message is to copy exisitng one from other GRIB file, +//! // modify the keys and write to new file. //! -//! if msg.read_key("level")?.value == KeyType::Int(900) { -//! if let KeyType::FloatArray(vals) = msg.read_key("values")?.value { -//! t900 = vals; -//! } -//! } -//! } -//! } +//! // Here we are computing the temperature at 850hPa as an average +//! // of 900hPa and 800hPa and writing it to a new file. //! -//! let mut new_msg = new_msg.remove(0); -//! -//! // Compute temperature at 850hPa -//! let t850: Vec = t800 -//! .iter() -//! .zip(t900.iter()) -//! .map(|t| (t.0 + t.1) / 2.0) -//! .collect(); -//! -//! // Edit appropriate keys in the editable message -//! new_msg.write_key(Key { -//! name: "level".to_string(), -//! value: KeyType::Int(850), -//! })?; -//! new_msg.write_key(Key { -//! name: "values".to_string(), -//! value: KeyType::FloatArray(t850), -//! })?; -//! -//! // Save the message to a new file without appending -//! new_msg.write_to_file(Path::new("iceland-850.grib"), false)?; -//! -//! # remove_file(Path::new("iceland-850.grib")).unwrap(); -//! # Ok(()) -//! # } -//!``` +//! use eccodes::FallibleStreamingIterator; +//! use eccodes::{CodesHandle, Key, KeyType, ProductKind}; +//! # use std::{fs::remove_file, path::Path}; +//! +//! # fn main() -> anyhow::Result<()> { +//! // Start by opening the file and creating CodesHandle +//! let file_path = Path::new("./data/iceland-levels.grib"); +//! let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; +//! +//! // We need a message to edit,in this case we can use +//! // temperature at 700hPa, which is similar to our result +//! let mut new_msg = vec![]; //! -//!### Features +//! // Get data values of temperatures at 800hPa and 900hPa +//! let mut t800 = vec![]; +//! let mut t900 = vec![]; //! -//!- `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation -//!on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! // Iterate over the messages and collect the data to defined vectors +//! while let Some(msg) = handle.next()? { +//! if msg.read_key("shortName")?.value == KeyType::Str("t".to_string()) { +//! if msg.read_key("level")?.value == KeyType::Int(700) { +//! // To use message outside of the iterator we need to clone it +//! new_msg.push(msg.try_clone()?); +//! } //! -//!To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` +//! if msg.read_key("level")?.value == KeyType::Int(800) { +//! if let KeyType::FloatArray(vals) = msg.read_key("values")?.value { +//! t800 = vals; +//! } +//! } //! -//!```text -//![package.metadata.docs.rs] -//!features = ["eccodes/docs"] -//!``` +//! if msg.read_key("level")?.value == KeyType::Int(900) { +//! if let KeyType::FloatArray(vals) = msg.read_key("values")?.value { +//! t900 = vals; +//! } +//! } +//! } +//! } //! - +//! // This converts the vector to a single message +//! let mut new_msg = new_msg.remove(0); +//! +//! // Compute temperature at 850hPa +//! let t850: Vec = t800 +//! .iter() +//! .zip(t900.iter()) +//! .map(|t| (t.0 + t.1) / 2.0) +//! .collect(); +//! +//! // Edit appropriate keys in the editable message +//! new_msg.write_key(Key { +//! name: "level".to_string(), +//! value: KeyType::Int(850), +//! })?; +//! new_msg.write_key(Key { +//! name: "values".to_string(), +//! value: KeyType::FloatArray(t850), +//! })?; +//! +//! // Save the message to a new file without appending +//! new_msg.write_to_file(Path::new("iceland-850.grib"), false)?; +//! +//! # remove_file(Path::new("iceland-850.grib")).unwrap(); +//! # Ok(()) +//! # } +//! ``` +//! + pub mod codes_handle; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 7d852ac..8f211c4 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -1,8 +1,13 @@ +#![cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +//! Definition of functions to convert a `KeyedMessage` to ndarray + use ndarray::{s, Array2, Array3}; use crate::{errors::MessageNdarrayError, CodesError, KeyType, KeyedMessage}; +/// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method #[derive(Clone, PartialEq, Debug, Default)] +#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] pub struct RustyCodesMessage { pub longitudes: Array2, pub latitudes: Array2, @@ -10,9 +15,21 @@ pub struct RustyCodesMessage { } impl KeyedMessage { - /// Returns [y, x] ([Nj, Ni], [lat, lon]) ndarray from the message, + /// Converts the message to a 2D ndarray. + /// + /// Returns ndarray where first dimension represents y coordinates and second dimension represents x coordinates, + /// ie. `[lat, lon]`. Index `[0, 0]` is the top-left corner of the grid: /// x coordinates are increasing with the i index, /// y coordinates are decreasing with the j index. + /// + /// Requires the keys `Ni`, `Nj` and `values` to be present in the message. + /// + /// Tested only with simple lat-lon grids. + /// + /// # Errors + /// + /// - When the required keys are not present or if their values are not of the expected type + /// - When the number of values mismatch with the `Ni` and `Nj` keys #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] pub fn to_ndarray(&self) -> Result, CodesError> { let KeyType::Int(ni) = self.read_key("Ni")?.value else { @@ -40,6 +57,17 @@ impl KeyedMessage { Ok(vals) } + /// Same as [`KeyedMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. + /// Fields are returned as separate arrays in [`RustyCodesMessage`]. + /// + /// Compared to `to_ndarray` this method has performance overhead as returned arrays may be cloned. + /// + /// This method requires the `latLonValues`, `Ni` and Nj` keys to be present in the message. + /// + /// # Errors + /// + /// - When the required keys are not present or if their values are not of the expected type + /// - When the number of values mismatch with the `Ni` and `Nj` keys #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] pub fn to_lons_lats_values(&self) -> Result { let KeyType::Int(ni) = self.read_key("Ni")?.value else { diff --git a/tests/index.rs b/tests/index.rs index 2249b33..2e738e3 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -73,7 +73,7 @@ fn collect_index_iterator() -> Result<()> { let mut levels = vec![]; while let Some(msg) = handle.next()? { - levels.push(msg.clone()); + levels.push(msg.try_clone()?); } assert_eq!(levels.len(), 5);