diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml index 76ae3c2..5ab6fe2 100644 --- a/.github/workflows/rust-dev.yml +++ b/.github/workflows/rust-dev.yml @@ -23,13 +23,13 @@ jobs: cargo clean - name: Test with cargo run: | - RUST_BACKTRACE=full cargo test + RUST_BACKTRACE=full cargo test --features "experimental_index" - name: Check with clippy run: | - cargo clippy -- -W clippy::pedantic + cargo clippy --features "experimental_index" - name: Build release run: | - cargo build --release + cargo build --release --features "experimental_index" build-macos: @@ -45,10 +45,10 @@ jobs: cargo clean - name: Test with cargo run: | - RUST_BACKTRACE=full cargo test + RUST_BACKTRACE=full cargo test --features "experimental_index" - name: Check with clippy run: | - cargo clippy -- -W clippy::pedantic + cargo clippy --features "experimental_index" - name: Build release run: | - cargo build --release + cargo build --release --features "experimental_index" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4431fb8..bb91cc0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -27,11 +27,11 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release + cargo build --release --features "experimental_index" cargo clean - name: Test with cargo run: | - cargo test + cargo test --features "experimental_index" cargo clean - name: Benchmark with criterion run: | @@ -53,11 +53,11 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release + cargo build --release --features "experimental_index" cargo clean - name: Test with cargo run: | - cargo test + cargo test --features "experimental_index" cargo clean - name: Benchmark with criterion run: | diff --git a/Cargo.toml b/Cargo.toml index c89ccc2..b7876e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" -version = "0.8.0" +version = "0.9.0" readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] @@ -18,7 +18,7 @@ edition = "2021" exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] [dependencies] -eccodes-sys = "0.5.1" +eccodes-sys = "0.5.2" libc = "0.2" thiserror = "1.0" bytes = "1.5" @@ -29,17 +29,18 @@ num-traits = "0.2" fallible-iterator = "0.3" [dev-dependencies] -eccodes-sys = "0.5.1" reqwest = { version = "0.11", features = ["rustls-tls"] } tokio = { version = "1.35", features = ["macros", "rt"] } criterion = "0.5" testing_logger = "0.1" +rand = "0.8" [features] docs = ["eccodes-sys/docs"] +experimental_index = [] [package.metadata.docs.rs] -features = ["docs"] +features = ["docs", "experimental_index"] [[bench]] name = "main" diff --git a/data/iceland-surface.idx b/data/iceland-surface.idx new file mode 100644 index 0000000..2aae480 Binary files /dev/null and b/data/iceland-surface.idx differ diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index e4c4d57..f9d9fea 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -9,6 +9,10 @@ use crate::{ codes_handle_new_from_message_copy, }, }; +#[cfg(feature = "experimental_index")] +use crate::{intermediate_bindings::codes_index::codes_handle_new_from_index, CodesIndex}; + +use super::GribFile; ///`FallibleIterator` implementation for `CodesHandle` to access GRIB messages inside file. /// @@ -18,7 +22,7 @@ use crate::{ ///Therefore this crate utilizes the `Iterator` to provide the access to GRIB messages in ///a safe and convienient way. /// -///[`FallibleIterator`](fallible_iterator::FallibleIterator) is used instead of classic `Iterator` +///[`FallibleIterator`] is used instead of classic `Iterator` ///because internal ecCodes functions can return error codes when the GRIB file ///is corrupted and for some other reasons. The usage of `FallibleIterator` is sligthly different ///than usage of `Iterator`, check its documentation for more details. @@ -77,23 +81,52 @@ use crate::{ ///## Errors ///The `next()` method will return [`CodesInternal`](crate::errors::CodesInternal) ///when internal ecCodes function returns non-zero code. -impl FallibleIterator for CodesHandle { +impl FallibleIterator for CodesHandle { + type Item = KeyedMessage; + + type Error = CodesError; + + fn next(&mut self) -> Result, Self::Error> { + let new_eccodes_handle; + unsafe { + codes_handle_delete(self.eccodes_handle)?; + new_eccodes_handle = codes_handle_new_from_file(self.source.pointer, self.product_kind); + } + + match new_eccodes_handle { + Ok(h) => { + self.eccodes_handle = h; + + if self.eccodes_handle.is_null() { + Ok(None) + } else { + let message = get_message_from_handle(h); + Ok(Some(message)) + } + } + Err(e) => Err(e), + } + } +} + +#[cfg(feature = "experimental_index")] +impl FallibleIterator for CodesHandle { type Item = KeyedMessage; type Error = CodesError; fn next(&mut self) -> Result, Self::Error> { - let file_handle; + let new_eccodes_handle; unsafe { - codes_handle_delete(self.file_handle)?; - file_handle = codes_handle_new_from_file(self.file_pointer, self.product_kind); + codes_handle_delete(self.eccodes_handle)?; + new_eccodes_handle = codes_handle_new_from_index(self.source.pointer); } - match file_handle { + match new_eccodes_handle { Ok(h) => { - self.file_handle = h; + self.eccodes_handle = h; - if self.file_handle.is_null() { + if self.eccodes_handle.is_null() { Ok(None) } else { let message = get_message_from_handle(h); diff --git a/src/codes_handle/keyed_message/iterator.rs b/src/codes_handle/keyed_message/iterator.rs index 751c49f..5e980fa 100644 --- a/src/codes_handle/keyed_message/iterator.rs +++ b/src/codes_handle/keyed_message/iterator.rs @@ -18,7 +18,7 @@ use super::KeysIteratorFlags; ///so it is probably more efficient to call that function directly only for keys you ///are interested in. /// -///[`FallibleIterator`](fallible_iterator::FallibleIterator) is used instead of classic `Iterator` +///[`FallibleIterator`] is used instead of classic `Iterator` ///because internal ecCodes functions can return internal error in some edge-cases. ///The usage of `FallibleIterator` is sligthly different than usage of `Iterator`, ///check its documentation for more details. diff --git a/src/codes_handle/keyed_message/mod.rs b/src/codes_handle/keyed_message/mod.rs index 2ca6237..f9c3d05 100644 --- a/src/codes_handle/keyed_message/mod.rs +++ b/src/codes_handle/keyed_message/mod.rs @@ -122,7 +122,7 @@ impl Drop for KeyedMessage { ///Technical note: delete functions in ecCodes can only fail with [`CodesInternalError`](crate::errors::CodesInternal::CodesInternalError) ///when other functions corrupt the inner memory of pointer, in that case memory leak is possible. ///In case of corrupt pointer segmentation fault will occur. - ///The pointers are cleared at the end of drop as they ar not not functional despite the result of delete functions. + ///The pointers are cleared at the end of drop as they are not functional despite the result of delete functions. fn drop(&mut self) { if let Some(nrst) = self.nearest_handle { unsafe { diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 1c980cc..841daec 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -1,13 +1,16 @@ //!Main crate module containing definition of `CodesHandle` //!and all associated functions and data structures -use crate::errors::CodesError; +#[cfg(feature = "experimental_index")] +use crate::{codes_index::CodesIndex, intermediate_bindings::codes_index::codes_index_delete}; +use crate::{errors::CodesError, intermediate_bindings::codes_handle_delete}; use bytes::Bytes; use eccodes_sys::{codes_handle, codes_keys_iterator, codes_nearest, ProductKind_PRODUCT_GRIB}; use errno::errno; use libc::{c_char, c_void, size_t, FILE}; use log::warn; use std::{ + fmt::Debug, fs::{File, OpenOptions}, os::unix::prelude::AsRawFd, path::Path, @@ -24,14 +27,20 @@ use eccodes_sys::{ mod iterator; mod keyed_message; +#[derive(Debug)] +#[doc(hidden)] +pub struct GribFile { + pointer: *mut FILE, +} + ///Main structure used to operate on the GRIB file. ///It takes a full ownership of the accessed file. ///It can be constructed either using a file or a memory buffer. #[derive(Debug)] -pub struct CodesHandle { - file_handle: *mut codes_handle, +pub struct CodesHandle { + eccodes_handle: *mut codes_handle, _data: DataContainer, - file_pointer: *mut FILE, + source: SOURCE, product_kind: ProductKind, } @@ -109,6 +118,8 @@ pub enum KeysIteratorFlags { enum DataContainer { FileBytes(Bytes), FileBuffer(File), + #[cfg(feature = "experimental_index")] + Empty(), } ///Enum representing the kind of product (file type) inside handled file. @@ -134,7 +145,7 @@ pub struct NearestGridpoint { pub value: f64, } -impl CodesHandle { +impl CodesHandle { ///The constructor that takes a [`path`](Path) to an existing file and ///a requested [`ProductKind`] and returns the [`CodesHandle`] object. /// @@ -168,7 +179,7 @@ impl CodesHandle { ///when the stream cannot be created from the file descriptor. /// ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`](eccodes_sys::codes_handle) cannot be created. + ///when internal [`codes_handle`] cannot be created. /// ///Returns [`CodesError::NoMessages`] when there is no message of requested type ///in the provided file. @@ -180,8 +191,10 @@ impl CodesHandle { Ok(CodesHandle { _data: (DataContainer::FileBuffer(file)), - file_handle, - file_pointer, + eccodes_handle: file_handle, + source: GribFile { + pointer: file_pointer, + }, product_kind, }) } @@ -220,7 +233,7 @@ impl CodesHandle { ///when the file stream cannot be created. /// ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`](eccodes_sys::codes_handle) cannot be created. + ///when internal [`codes_handle`] cannot be created. /// ///Returns [`CodesError::NoMessages`] when there is no message of requested type ///in the provided file. @@ -230,17 +243,39 @@ impl CodesHandle { ) -> Result { let file_pointer = open_with_fmemopen(&file_data)?; - let file_handle = null_mut(); + let eccodes_handle = null_mut(); Ok(CodesHandle { _data: (DataContainer::FileBytes(file_data)), - file_handle, - file_pointer, + eccodes_handle, + source: GribFile { + pointer: file_pointer, + }, product_kind, }) } } +#[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +impl CodesHandle { + pub fn new_from_index( + index: CodesIndex, + product_kind: ProductKind, + ) -> Result { + let eccodes_handle = null_mut(); + + let new_handle = CodesHandle { + _data: DataContainer::Empty(), //unused, index owns data + eccodes_handle, + source: index, + product_kind, + }; + + Ok(new_handle) + } +} + fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { let file_ptr; unsafe { @@ -275,7 +310,51 @@ fn open_with_fmemopen(file_data: &Bytes) -> Result<*mut FILE, CodesError> { Ok(file_ptr) } -impl Drop for CodesHandle { +/// This trait is neccessary because (1) drop in GribFile/IndexFile cannot +/// be called directly as source cannot be moved out of shared reference +/// and (2) Drop drops fields in arbitrary order leading to fclose() failing +#[doc(hidden)] +pub trait SpecialDrop { + fn spec_drop(&mut self); +} + +impl SpecialDrop for GribFile { + fn spec_drop(&mut self) { + //fclose() can fail in several different cases, however there is not much + //that we can nor we should do about it. the promise of fclose() is that + //the stream will be disassociated from the file after the call, therefore + //use of stream after the call to fclose() is undefined behaviour, so we clear it + let return_code; + unsafe { + if !self.pointer.is_null() { + return_code = libc::fclose(self.pointer); + if return_code != 0 { + let error_val = errno(); + warn!( + "fclose() returned an error and your file might not have been correctly saved. + Error code: {}; Error message: {}", + error_val.0, error_val + ); + } + } + } + + self.pointer = null_mut(); + } +} + +#[cfg(feature = "experimental_index")] +impl SpecialDrop for CodesIndex { + fn spec_drop(&mut self) { + unsafe { + codes_index_delete(self.pointer); + } + + self.pointer = null_mut(); + } +} + +impl Drop for CodesHandle { ///Executes the destructor for this type. ///This method calls `fclose()` from libc for graceful cleanup. /// @@ -287,25 +366,15 @@ impl Drop for CodesHandle { ///If any function called in the destructor returns an error warning will appear in log. ///If bugs occurs during `CodesHandle` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). fn drop(&mut self) { - //fclose() can fail in several different cases, however there is not much - //that we can nor we should do about it. the promise of fclose() is that - //the stream will be disassociated from the file after the call, therefore - //use of stream after the call to fclose() is undefined behaviour, so we clear it - let return_code; unsafe { - return_code = libc::fclose(self.file_pointer); + codes_handle_delete(self.eccodes_handle).unwrap_or_else(|error| { + warn!("codes_handle_delete() returned an error: {:?}", &error); + }); } - if return_code != 0 { - let error_val = errno(); - warn!( - "fclose() returned an error and your file might not have been correctly saved. - Error code: {}; Error message: {}", - error_val.0, error_val - ); - } + self.eccodes_handle = null_mut(); - self.file_pointer = null_mut(); + self.source.spec_drop(); } } @@ -314,6 +383,8 @@ mod tests { use eccodes_sys::ProductKind_PRODUCT_GRIB; use crate::codes_handle::{CodesHandle, DataContainer, ProductKind}; + #[cfg(feature = "experimental_index")] + use crate::codes_index::{CodesIndex, Select}; use log::Level; use std::path::Path; @@ -324,13 +395,13 @@ mod tests { let handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - assert!(!handle.file_pointer.is_null()); - assert!(handle.file_handle.is_null()); + assert!(!handle.source.pointer.is_null()); + assert!(handle.eccodes_handle.is_null()); assert_eq!(handle.product_kind as u32, { ProductKind_PRODUCT_GRIB }); let metadata = match &handle._data { - DataContainer::FileBytes(_) => panic!(), DataContainer::FileBuffer(file) => file.metadata().unwrap(), + _ => panic!(), }; println!("{:?}", metadata); @@ -349,16 +420,39 @@ mod tests { .unwrap(); let handle = CodesHandle::new_from_memory(file_data, product_kind).unwrap(); - assert!(!handle.file_pointer.is_null()); - assert!(handle.file_handle.is_null()); + assert!(!handle.source.pointer.is_null()); + assert!(handle.eccodes_handle.is_null()); assert_eq!(handle.product_kind as u32, { ProductKind_PRODUCT_GRIB }); match &handle._data { DataContainer::FileBytes(file) => assert!(!file.is_empty()), - DataContainer::FileBuffer(_) => panic!(), + _ => panic!(), }; } + #[test] + #[cfg(feature = "experimental_index")] + fn index_constructor_and_destructor() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + let i_ptr = index.pointer.clone(); + + let handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + + assert_eq!(handle.source.pointer, i_ptr); + assert!(handle.eccodes_handle.is_null()); + } + #[tokio::test] async fn codes_handle_drop() { testing_logger::setup(); diff --git a/src/codes_index/mod.rs b/src/codes_index/mod.rs new file mode 100644 index 0000000..1b79cb1 --- /dev/null +++ b/src/codes_index/mod.rs @@ -0,0 +1,160 @@ +//!Main crate module containing definition of `CodesIndex` +//!and all associated functions and data structures + +use crate::{ + codes_handle::SpecialDrop, + errors::CodesError, + intermediate_bindings::codes_index::{ + codes_index_add_file, codes_index_new, codes_index_read, codes_index_select_double, + codes_index_select_long, codes_index_select_string, + }, +}; +use eccodes_sys::codes_index; +use std::path::Path; + +#[derive(Debug)] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +pub struct CodesIndex { + pub(crate) pointer: *mut codes_index, +} +pub trait Select { + fn select(self, key: &str, value: T) -> Result; +} + +impl CodesIndex { + #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + pub fn new_from_keys(keys: &[&str]) -> Result { + let keys = keys.join(","); + + let index_handle; + unsafe { + // technically codes_index_new can also select keys + // but that would unnecessarily diverge the API + // and would be error prone + index_handle = codes_index_new(&keys)?; + } + Ok(CodesIndex { + pointer: index_handle, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + pub fn read_from_file(index_file_path: &Path) -> Result { + let file_path = index_file_path.to_str().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") + })?; + + let index_handle; + unsafe { + index_handle = codes_index_read(file_path)?; + } + + Ok(CodesIndex { + pointer: index_handle, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + pub fn add_grib_file(self, index_file_path: &Path) -> Result { + let file_path = index_file_path.to_str().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") + })?; + + let new_index = self; + + unsafe { + codes_index_add_file(new_index.pointer, file_path)?; + } + + Ok(new_index) + } +} + +impl Select for CodesIndex { + fn select(self, key: &str, value: i64) -> Result { + let new_index = self; + unsafe { + codes_index_select_long(new_index.pointer, key, value)?; + } + + Ok(new_index) + } +} +impl Select for CodesIndex { + fn select(self, key: &str, value: f64) -> Result { + let new_index = self; + unsafe { + codes_index_select_double(new_index.pointer, key, value)?; + } + Ok(new_index) + } +} +impl Select<&str> for CodesIndex { + fn select(self, key: &str, value: &str) -> Result { + let new_index = self; + unsafe { + codes_index_select_string(new_index.pointer, key, value)?; + } + Ok(new_index) + } +} + +impl Drop for CodesIndex { + fn drop(&mut self) { + self.spec_drop(); + } +} + +#[cfg(test)] +mod tests { + use crate::codes_index::{CodesIndex, Select}; + use std::path::Path; + #[test] + fn index_constructors() { + { + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + assert!(!index.pointer.is_null()); + } + { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path).unwrap(); + assert!(!index.pointer.is_null()); + } + } + + #[test] + fn index_destructor() { + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + + drop(index) + } + + #[test] + fn add_file() { + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + let grib_path = Path::new("./data/iceland.grib"); + let index = index.add_grib_file(grib_path).unwrap(); + + assert!(!index.pointer.is_null()); + } + + #[test] + fn index_selection() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + assert!(!index.pointer.is_null()); + } +} diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs new file mode 100644 index 0000000..c285810 --- /dev/null +++ b/src/intermediate_bindings/codes_index.rs @@ -0,0 +1,147 @@ +#![allow(non_camel_case_types)] +#![allow(clippy::module_name_repetitions)] + +use eccodes_sys::{codes_index, CODES_LOCK}; +use std::{ffi::CString, ptr}; + +#[cfg(target_os = "macos")] +type _SYS_IO_FILE = eccodes_sys::__sFILE; + +#[cfg(not(target_os = "macos"))] +type _SYS_IO_FILE = eccodes_sys::_IO_FILE; + +use eccodes_sys::{codes_context, codes_handle}; +use num_traits::FromPrimitive; + +use crate::errors::{CodesError, CodesInternal}; + +// all index functions are safeguarded by a lock +// because there are random errors appearing when using the index functions concurrently + +pub unsafe fn codes_index_new(keys: &str) -> Result<*mut codes_index, CodesError> { + let context: *mut codes_context = ptr::null_mut(); //default context + let mut error_code: i32 = 0; + let keys = CString::new(keys).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let codes_index = eccodes_sys::codes_index_new(context, keys.as_ptr(), &mut error_code); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(codes_index) +} + +pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, CodesError> { + let filename = CString::new(filename).unwrap(); + let context: *mut codes_context = ptr::null_mut(); //default context + let mut error_code: i32 = 0; + + let _g = CODES_LOCK.lock().unwrap(); + let codes_index = eccodes_sys::codes_index_read(context, filename.as_ptr(), &mut error_code); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(codes_index) +} + +pub unsafe fn codes_index_delete(index: *mut codes_index) { + if index.is_null() { + return; + } + + let _g = CODES_LOCK.lock().unwrap(); + eccodes_sys::codes_index_delete(index); +} + +pub unsafe fn codes_index_add_file( + index: *mut codes_index, + filename: &str, +) -> Result<(), CodesError> { + let filename = CString::new(filename).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_add_file(index, filename.as_ptr()); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_index_select_long( + index: *mut codes_index, + key: &str, + value: i64, +) -> Result<(), CodesError> { + let key = CString::new(key).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_select_long(index, key.as_ptr(), value); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_index_select_double( + index: *mut codes_index, + key: &str, + value: f64, +) -> Result<(), CodesError> { + let key = CString::new(key).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_select_double(index, key.as_ptr(), value); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_index_select_string( + index: *mut codes_index, + key: &str, + value: &str, +) -> Result<(), CodesError> { + let key = CString::new(key).unwrap(); + let value = CString::new(value).unwrap(); + + let _g = CODES_LOCK.lock().unwrap(); + let error_code = eccodes_sys::codes_index_select_string(index, key.as_ptr(), value.as_ptr()); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(()) +} + +pub unsafe fn codes_handle_new_from_index( + index: *mut codes_index, +) -> Result<*mut codes_handle, CodesError> { + let mut error_code: i32 = 0; + + let _g = CODES_LOCK.lock().unwrap(); + let codes_handle = eccodes_sys::codes_handle_new_from_index(index, &mut error_code); + + // special case! codes_handle_new_from_index returns -43 when there are no messages left in the index + // this is also indicated by a null pointer, which is handled upstream + if error_code == -43 { + return Ok(codes_handle); + } + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + Ok(codes_handle) +} diff --git a/src/intermediate_bindings.rs b/src/intermediate_bindings/mod.rs similarity index 99% rename from src/intermediate_bindings.rs rename to src/intermediate_bindings/mod.rs index 0397199..cf7f521 100644 --- a/src/intermediate_bindings.rs +++ b/src/intermediate_bindings/mod.rs @@ -7,6 +7,9 @@ //!to make ecCodes usage safer and easier, //!but they are unsafe as they operate on raw `codes_handle`. +#[cfg(feature = "experimental_index")] +pub mod codes_index; + use std::{ ffi::{CStr, CString}, ptr::{self, addr_of_mut}, @@ -68,6 +71,10 @@ pub unsafe fn codes_handle_new_from_file( } pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { + if handle.is_null() { + return Ok(()); + } + let error_code = eccodes_sys::codes_handle_delete(handle); if error_code != 0 { diff --git a/src/lib.rs b/src/lib.rs index 0ae82a3..fb7e311 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,9 +25,9 @@ //!### Accessing GRIB files //! //!This crate provides an access to GRIB file by creating a -//![`CodesHandle`](codes_handle::CodesHandle) and reading messages from the file with it. +//![`CodesHandle`] and reading messages from the file with it. //! -//!The [`CodesHandle`](codes_handle::CodesHandle) can be constructed in two ways: +//!The [`CodesHandle`] can be constructed in two ways: //! //!- The main option is to use [`new_from_file()`](codes_handle::CodesHandle::new_from_file) function //!to open a file under provided [`path`](`std::path::Path`) with filesystem, @@ -41,8 +41,8 @@ //!Data (messages) inside the GRIB file can be accessed using the [`FallibleIterator`](`codes_handle::CodesHandle#impl-FallibleIterator`) //!by iterating over the `CodesHandle`. //! -//!The `FallibleIterator` returns a [`KeyedMessage`](codes_handle::KeyedMessage) structure which implements some -//!methods to access data values. The data inside `KeyedMessage` is provided directly as [`Key`](codes_handle::Key) +//!The `FallibleIterator` returns a [`KeyedMessage` structure which implements some +//!methods to access data values. The data inside `KeyedMessage` is provided directly as [`Key`] //!or as more specific data type. //! //!#### Example @@ -217,11 +217,17 @@ //! pub mod codes_handle; +#[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +pub mod codes_index; pub mod errors; mod intermediate_bindings; pub use codes_handle::{ CodesHandle, Key, KeyType, KeyedMessage, KeysIteratorFlags, NearestGridpoint, ProductKind, }; +#[cfg(feature = "experimental_index")] +#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] +pub use codes_index::CodesIndex; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; diff --git a/tests/handle.rs b/tests/handle.rs new file mode 100644 index 0000000..7be7f82 --- /dev/null +++ b/tests/handle.rs @@ -0,0 +1,51 @@ +use std::{path::Path, thread}; + +use eccodes::{CodesHandle, KeyType, ProductKind}; +use fallible_iterator::FallibleIterator; + +#[test] +fn thread_safety() { + thread::spawn(|| loop { + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB).unwrap(); + let current_message = handle.next().unwrap().unwrap(); + + for _ in 0..100 { + let _ = current_message.read_key("name").unwrap(); + + let str_key = current_message.read_key("name").unwrap(); + + match str_key.value { + KeyType::Str(_) => {} + _ => panic!("Incorrect variant of string key"), + } + + assert_eq!(str_key.name, "name"); + } + + drop(current_message); + drop(handle); + }); + + for _ in 0..1000 { + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB).unwrap(); + let current_message = handle.next().unwrap().unwrap(); + + let long_key = current_message + .read_key("numberOfPointsAlongAParallel") + .unwrap(); + + match long_key.value { + KeyType::Int(_) => {} + _ => panic!("Incorrect variant of long key"), + } + + assert_eq!(long_key.name, "numberOfPointsAlongAParallel"); + + drop(current_message); + drop(handle); + } +} diff --git a/tests/index.rs b/tests/index.rs new file mode 100644 index 0000000..69d0792 --- /dev/null +++ b/tests/index.rs @@ -0,0 +1,261 @@ +#![cfg(feature = "experimental_index")] + +use std::{path::Path, thread}; + +use eccodes::{codes_index::Select, CodesHandle, CodesIndex, KeyType, ProductKind}; +use fallible_iterator::FallibleIterator; +use rand::Rng; + +#[test] +fn iterate_handle_from_index() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + let handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + + let counter = handle.count().unwrap(); + + assert_eq!(counter, 1); +} + +#[test] +fn read_index_messages() { + let file_path = Path::new("./data/iceland-surface.idx"); + let index = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + let mut handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + let current_message = handle.next().unwrap().unwrap(); + + { + let short_name = current_message.read_key("shortName").unwrap(); + match short_name.value { + KeyType::Str(val) => assert!(val == "2t"), + _ => panic!("Unexpected key type"), + }; + } + { + let level = current_message.read_key("level").unwrap(); + match level.value { + KeyType::Int(val) => assert!(val == 0), + _ => panic!("Unexpected key type"), + }; + } +} + +#[test] +fn collect_index_iterator() { + let keys = vec!["typeOfLevel", "level"]; + let index = CodesIndex::new_from_keys(&keys).unwrap(); + let grib_path = Path::new("./data/iceland-levels.grib"); + + let index = index + .add_grib_file(grib_path) + .unwrap() + .select("typeOfLevel", "isobaricInhPa") + .unwrap() + .select("level", 700) + .unwrap(); + + let handle = CodesHandle::new_from_index(index, ProductKind::GRIB).unwrap(); + + let level = handle.collect::>().unwrap(); + + assert_eq!(level.len(), 5); +} + +#[test] +fn add_file_error() { + thread::spawn(|| { + let grib_path = Path::new("./data/iceland-levels.grib"); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let mut index_op = CodesIndex::new_from_keys(&keys).unwrap(); + + loop { + index_op = index_op.add_grib_file(grib_path).unwrap(); + } + }); + + thread::sleep(std::time::Duration::from_millis(250)); + + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let wrong_path = Path::new("./data/xxx.grib"); + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(wrong_path); + + assert!(index.is_err()); +} + +#[test] +fn index_panic() { + thread::spawn(|| { + let grib_path = Path::new("./data/iceland-levels.grib"); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let mut index_op = CodesIndex::new_from_keys(&keys).unwrap(); + + loop { + index_op = index_op.add_grib_file(grib_path).unwrap(); + } + }); + + thread::sleep(std::time::Duration::from_millis(250)); + + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let wrong_path = Path::new("./data/xxx.grib"); + let index = CodesIndex::new_from_keys(&keys).unwrap(); + + let result = std::panic::catch_unwind(|| index.add_grib_file(wrong_path).unwrap()); + + assert!(result.is_err()); +} + +#[test] +fn add_file_while_index_open() { + thread::spawn(|| { + let file_path = Path::new("./data/iceland-surface.idx"); + let mut index_op = CodesIndex::read_from_file(file_path).unwrap(); + + loop { + index_op = index_op + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + } + }); + + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let grib_path = Path::new("./data/iceland-surface.grib"); + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(grib_path); + + assert!(index.is_ok()); +} + +#[test] +fn add_file_to_read_index() { + let file_path = Path::new("./data/iceland-surface.idx"); + let grib_path = Path::new("./data/iceland-surface.grib"); + + let _index = CodesIndex::read_from_file(file_path) + .unwrap() + .add_grib_file(grib_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); +} + +#[test] +fn simulatenous_index_destructors() { + let h1 = thread::spawn(|| { + let mut rng = rand::thread_rng(); + let file_path = Path::new("./data/iceland-surface.idx"); + + for _ in 0..10 { + let sleep_time = rng.gen_range(1..30); // randomizing sleep time to hopefully catch segfaults + + let index_op = CodesIndex::read_from_file(file_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + thread::sleep(std::time::Duration::from_millis(sleep_time)); + drop(index_op); + } + }); + + let h2 = thread::spawn(|| { + let mut rng = rand::thread_rng(); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let grib_path = Path::new("./data/iceland-surface.grib"); + + for _ in 0..10 { + let sleep_time = rng.gen_range(1..42); // randomizing sleep time to hopefully catch segfaults + + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(grib_path) + .unwrap() + .select("shortName", "2t") + .unwrap() + .select("typeOfLevel", "surface") + .unwrap() + .select("level", 0) + .unwrap() + .select("stepType", "instant") + .unwrap(); + + thread::sleep(std::time::Duration::from_millis(sleep_time)); + drop(index); + } + }); + + h1.join().unwrap(); + h2.join().unwrap(); +} + +#[test] +fn index_handle_interference() { + thread::spawn(|| { + let file_path = Path::new("./data/iceland.grib"); + + loop { + let handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB); + + assert!(handle.is_ok()); + } + }); + + let mut rng = rand::thread_rng(); + let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; + let grib_path = Path::new("./data/iceland.grib"); + + for _ in 0..10 { + let sleep_time = rng.gen_range(1..42); // randomizing sleep time to hopefully catch segfaults + + let index = CodesIndex::new_from_keys(&keys) + .unwrap() + .add_grib_file(grib_path) + .unwrap(); + let i_handle = CodesHandle::new_from_index(index, ProductKind::GRIB); + + assert!(i_handle.is_ok()); + + thread::sleep(std::time::Duration::from_millis(sleep_time)); + } +}