Skip to content

Commit

Permalink
Minor updates (#19)
Browse files Browse the repository at this point in the history 
* add checking for j scaning in ndarray

* remove downloading from tests (except doc tests)

* explicitly allow data containters to be unused

* update deps and remove implicit features
  • Loading branch information
@Quba1
Quba1 authored Apr 9, 2024
1 parent 0f9afb1 commit 8fbf6be
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 72 deletions.
11 changes: 5 additions & 6 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
name = "eccodes"
description = "Unofficial high-level Rust bindings of the latest ecCodes release"
repository = "https://github.com/ScaleWeather/eccodes"
version = "0.10.1"
version = "0.10.3"
readme = "README.md"
authors = ["Jakub Lewandowski <[email protected]>"]
keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"]
Expand All @@ -22,10 +22,10 @@ rust-version = "1.70.0"
eccodes-sys = { version = "0.5.2", default-features = false }
libc = { version = "0.2", default-features = false }
thiserror = { version = "1.0", default-features = false }
bytes = { version = "1.5", default-features = false }
bytes = { version = "1.6", default-features = false }
log = { version = "0.4", default-features = false }
errno = { version = "0.3", default-features = false }
num-derive = { version = "0.4.1", default-features = false }
num-derive = { version = "0.4", default-features = false }
num-traits = { version = "0.2", default-features = false }
fallible-iterator = { version = "0.3", default-features = false }
fallible-streaming-iterator = { version = "0.1.9", default-features = false }
Expand All @@ -34,8 +34,7 @@ ndarray = { version = "0.15", default-features = false, optional = true, feature
] }

[dev-dependencies]
reqwest = { version = "0.11", features = ["rustls-tls"] }
tokio = { version = "1.35", features = ["macros", "rt"] }
reqwest = { version = "0.12", features = ["rustls-tls"] }
criterion = "0.5"
testing_logger = "0.1"
rand = "0.8"
Expand All @@ -46,7 +45,7 @@ float-cmp = "0.9"
default = ["message_ndarray"]
docs = ["eccodes-sys/docs"]
experimental_index = []
message_ndarray = ["ndarray"]
message_ndarray = ["dep:ndarray"]

[package.metadata.docs.rs]
features = ["docs", "experimental_index", "message_ndarray"]
Expand Down
93 changes: 46 additions & 47 deletions src/codes_handle/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,55 +34,55 @@ pub struct GribFile {
/// - File path using [`new_from_file()`](CodesHandle::new_from_file)
/// - From memory buffer using [`new_from_memory()`](CodesHandle::new_from_memory)
/// - From GRIB index using [`new_from_index()`](CodesHandle::new_from_index) (with `experimental_index` feature enabled)
///
///
/// Destructor for this structure does not panic, but some internal functions may rarely fail
/// leading to bugs. Errors encountered in the destructor are logged with [`log`].
///
///
/// # `FallibleStreamingIterator`
///
///
/// This structure implements [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait which allows to access GRIB messages.
///
///
/// To access GRIB messages the ecCodes library uses a method similar to a C-style iterator.
/// It digests the `* FILE` multiple times, each time returning the `*mut codes_handle`
/// to a message inside the file. The behavior of previous `*mut codes_handle` after next one is generated is undefined
/// and we assume here that it is unsafe to use "old" `*mut codes_handle`.
///
///
/// In Rust, such pattern is best represented by a streaming iterator which returns a reference to the message,
/// that is valid only until the next iteration. If you need to prolong the lifetime of the message, you can clone it.
/// Internal ecCodes functions can fail, necessitating the streaming iterator to be implemented with
/// Internal ecCodes functions can fail, necessitating the streaming iterator to be implemented with
/// [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait.
///
///
/// As of `0.10` release, none of the available streaming iterator crates utilises already stabilized GATs.
/// This unfortunately significantly limits the number of methods available for `CodesHandle` iterator.
/// Therefore the probably most versatile way to iterate over the messages is to use `while let` loop.
///
///
/// ```
/// use eccodes::{ProductKind, CodesHandle, KeyType};
/// # use std::path::Path;
/// // FallibleStreamingIterator must be in scope to use it
/// // FallibleStreamingIterator must be in scope to use it
/// use eccodes::FallibleStreamingIterator;
/// #
/// # fn main() -> anyhow::Result<(), eccodes::errors::CodesError> {
/// let file_path = Path::new("./data/iceland-surface.grib");
/// let product_kind = ProductKind::GRIB;
///
///
/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?;
///
///
/// // Print names of messages in the file
/// while let Some(message) = handle.next()? {
/// // The message must be unwraped as internal next() can fail
/// let key = message.read_key("name")?;
///
///
/// if let KeyType::Str(name) = key.value {
/// println!("{:?}", name);
/// }
/// }
/// # Ok(())
/// # }
/// ```
///
///
/// You can also manually collect the messages into a vector to use them later.
///
///
/// ```
/// use eccodes::{ProductKind, CodesHandle, KeyedMessage};
/// # use eccodes::errors::CodesError;
Expand All @@ -92,18 +92,18 @@ pub struct GribFile {
/// # fn main() -> anyhow::Result<(), eccodes::errors::CodesError> {
/// let file_path = Path::new("./data/iceland-surface.grib");
/// let product_kind = ProductKind::GRIB;
///
///
/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?;
///
///
/// let mut handle_collected = vec![];
///
///
/// while let Some(msg) = handle.next()? {
/// handle_collected.push(msg.try_clone()?);
/// }
/// # Ok(())
/// # }
/// ```
///
///
/// All available methods for `CodesHandle` iterator can be found in [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait.
#[derive(Debug)]
pub struct CodesHandle<SOURCE: Debug + SpecialDrop> {
Expand All @@ -115,7 +115,9 @@ pub struct CodesHandle<SOURCE: Debug + SpecialDrop> {

#[derive(Debug)]
enum DataContainer {
#[allow(unused)]
FileBytes(Bytes),
#[allow(unused)]
FileBuffer(File),
#[cfg(feature = "experimental_index")]
Empty(),
Expand Down Expand Up @@ -237,36 +239,34 @@ impl CodesHandle<GribFile> {
#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))]
impl CodesHandle<CodesIndex> {
/// Creates [`CodesHandle`] for provided [`CodesIndex`].
///
///
/// ## Example
///
///
/// ```
/// # fn run() -> anyhow::Result<()> {
/// # use eccodes::{CodesHandle, CodesIndex};
/// #
/// let index = CodesIndex::new_from_keys(&vec!["shortName", "typeOfLevel", "level"])?;
/// let handle = CodesHandle::new_from_index(index)?;
///
///
/// Ok(())
/// # }
/// ```
///
///
/// The function takes ownership of the provided [`CodesIndex`] which owns
/// the GRIB data. [`CodesHandle`] created from [`CodesIndex`] is of different type
/// than the one created from file or memory buffer, because it internally uses
/// different functions to access messages. But it can be used in the same way.
///
///
/// ⚠️ Warning: This function may interfere with other functions in concurrent context,
/// due to ecCodes issues with thread-safety for indexes. More information can be found
/// in [`codes_index`](crate::codes_index) module documentation.
///
///
/// ## Errors
///
///
/// Returns [`CodesError::Internal`] with error code
/// when internal [`codes_handle`](eccodes_sys::codes_handle) cannot be created.
pub fn new_from_index(
index: CodesIndex,
) -> Result<Self, CodesError> {
pub fn new_from_index(index: CodesIndex) -> Result<Self, CodesError> {
let new_handle = CodesHandle {
_data: DataContainer::Empty(), //unused, index owns data
source: index,
Expand Down Expand Up @@ -363,12 +363,12 @@ impl SpecialDrop for CodesIndex {
#[doc(hidden)]
impl<S: Debug + SpecialDrop> Drop for CodesHandle<S> {
/// Executes the destructor for this type.
///
///
/// Currently it is assumed that under normal circumstances this destructor never fails.
/// However in some edge cases fclose can return non-zero code.
/// In such case all pointers and file descriptors are safely deleted.
/// However memory leaks can still occur.
///
///
/// If any function called in the destructor returns an error warning will appear in log.
/// If bugs occurs during `CodesHandle` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes).
fn drop(&mut self) {
Expand All @@ -379,13 +379,14 @@ impl<S: Debug + SpecialDrop> Drop for CodesHandle<S> {
#[cfg(test)]
mod tests {
use anyhow::Result;
use bytes::Bytes;
use eccodes_sys::ProductKind_PRODUCT_GRIB;

use crate::codes_handle::{CodesHandle, DataContainer, ProductKind};
#[cfg(feature = "experimental_index")]
use crate::codes_index::{CodesIndex, Select};
use log::Level;
use std::path::Path;
use std::{fs::File, io::Read, path::Path};

#[test]
fn file_constructor() -> Result<()> {
Expand All @@ -406,15 +407,14 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn memory_constructor() -> Result<()> {
#[test]
fn memory_constructor() -> Result<()> {
let product_kind = ProductKind::GRIB;
let file_data = reqwest::get(
"https://github.com/ScaleWeather/eccodes/blob/main/data/iceland.grib?raw=true",
)
.await?
.bytes()
.await?;

let mut f = File::open(Path::new("./data/iceland.grib"))?;
let mut buf = Vec::new();
f.read_to_end(&mut buf)?;
let file_data = Bytes::from(buf);

let handle = CodesHandle::new_from_memory(file_data, product_kind)?;
assert!(!handle.source.pointer.is_null());
Expand Down Expand Up @@ -451,8 +451,8 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn codes_handle_drop() -> Result<()> {
#[test]
fn codes_handle_drop() -> Result<()> {
testing_logger::setup();

{
Expand All @@ -469,12 +469,11 @@ mod tests {

{
let product_kind = ProductKind::GRIB;
let file_data = reqwest::get(
"https://github.com/ScaleWeather/eccodes/blob/main/data/iceland.grib?raw=true",
)
.await?
.bytes()
.await?;

let mut f = File::open(Path::new("./data/iceland.grib"))?;
let mut buf = Vec::new();
f.read_to_end(&mut buf)?;
let file_data = Bytes::from(buf);

let handle = CodesHandle::new_from_memory(file_data, product_kind)?;
drop(handle);
Expand Down
5 changes: 5 additions & 0 deletions src/errors.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ pub enum MessageNdarrayError {
#[error("The length of the values array ({0}) is different than expected ({1})")]
UnexpectedValuesLength(usize, usize),

/// Returned when functions converting to ndarray cannot correctly
/// parse key necessary for the conversion.
#[error("Requested key {0} has a value out of expected range")]
UnexpectedKeyValue(String),

/// Returned when ndarray cannot create an array with the shape
/// defined by Ni and Nj keys.
#[error("Error occured while converting to ndarray: {0}")]
Expand Down
Loading

0 comments on commit 8fbf6be

Please sign in to comment.