From 474b4f61d74a08185922f1e56c302f85b83fcbb9 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 4 Feb 2024 15:10:45 +0100 Subject: [PATCH 1/7] dont run dev actions on main --- .github/workflows/rust-dev.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml index 566ce0b..590b270 100644 --- a/.github/workflows/rust-dev.yml +++ b/.github/workflows/rust-dev.yml @@ -1,6 +1,12 @@ name: cargodev -on: [push, pull_request] +on: + push: + branches-ignore: + - main + pull_request: + branches: + - "**" env: CARGO_TERM_COLOR: always From a880f0b03181d8e7b22964531b25838cd07b6a14 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:02:37 +0100 Subject: [PATCH 2/7] initial ndarray implementation --- Cargo.toml | 3 ++ src/errors.rs | 25 ++++++++++++++ src/keys_iterator.rs | 6 ++-- src/lib.rs | 3 ++ src/message_ndarray.rs | 75 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 src/message_ndarray.rs diff --git a/Cargo.toml b/Cargo.toml index 7c51d5e..d47034f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ num-derive = "0.4.1" num-traits = "0.2" fallible-iterator = "0.3" fallible-streaming-iterator = "0.1.9" +ndarray = { version = "0.15", default-features = false, optional = true, features = ["std"]} [dev-dependencies] reqwest = { version = "0.11", features = ["rustls-tls"] } @@ -36,10 +37,12 @@ criterion = "0.5" testing_logger = "0.1" rand = "0.8" anyhow = "1.0" +float-cmp = "0.9" [features] docs = ["eccodes-sys/docs"] experimental_index = [] +message_ndarray = ["ndarray"] [package.metadata.docs.rs] features = ["docs", "experimental_index"] diff --git a/src/errors.rs b/src/errors.rs index 157cf30..df4014a 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -18,6 +18,12 @@ pub enum CodesError { #[error("ecCodes function returned a non-zero code {0}")] Internal(#[from] CodesInternal), + #[cfg(feature = "message_ndarray")] + /// Returned when function in `message_ndarray` module cannot convert + /// the message to ndarray. Check [`MessageNdarrayError`] for more details. + #[error("error occured while converting KeyedMessage to ndarray {0}")] + NdarrayConvert(#[from] MessageNdarrayError), + ///Returned when one of libc functions returns a non-zero error code. ///Check libc documentation for details of the errors. ///For libc reference check these websites: ([1](https://man7.org/linux/man-pages/index.html)) @@ -67,6 +73,25 @@ pub enum CodesError { NullPtr, } +#[cfg(feature = "message_ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[derive(Error, Debug)] +/// Errors returned by the `message_ndarray` module. +pub enum MessageNdarrayError { + /// Returned when functions converting to ndarray cannot correctly + /// read key necessary for the conversion. + #[error("Requested key {0} has a different type than expected")] + UnexpectedKeyType(String), + + /// Returned when length of values array is not equal to + /// product of Ni and Nj keys. + #[error("The length of the values array ({0}) is different than expected ({1})")] + UnexpectedValuesLength(usize, i64), + + #[error("Error occured while converting to ndarray: {0}")] + InvalidShape(#[from] ndarray::ShapeError), +} + #[derive(Copy, Eq, PartialEq, Clone, Ord, PartialOrd, Hash, Error, Debug, FromPrimitive)] ///Errors returned by internal ecCodes library functions. ///Copied directly from the ecCodes API. diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 532cc38..4f29c9a 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -90,7 +90,7 @@ impl KeyedMessage { pub fn new_keys_iterator( &self, flags: Vec, - namespace: String, + namespace: &str, ) -> Result { let flags = flags.iter().map(|f| *f as u32).sum(); @@ -215,7 +215,7 @@ mod tests { KeysIteratorFlags::SkipDuplicates, //32 ]; - let namespace = "geography".to_owned(); + let namespace = "geography"; let mut kiter = current_message.new_keys_iterator(flags, namespace)?; @@ -238,7 +238,7 @@ mod tests { KeysIteratorFlags::AllKeys, //0 ]; - let namespace = "blabla".to_owned(); + let namespace = "blabla"; let mut kiter = current_message.new_keys_iterator(flags, namespace)?; diff --git a/src/lib.rs b/src/lib.rs index a659d0c..a0b5024 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -225,6 +225,9 @@ pub mod errors; mod intermediate_bindings; pub mod keyed_message; pub mod keys_iterator; +#[cfg(feature = "message_ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +pub mod message_ndarray; mod pointer_guard; pub use codes_handle::{CodesHandle, ProductKind}; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs new file mode 100644 index 0000000..a5b1ae5 --- /dev/null +++ b/src/message_ndarray.rs @@ -0,0 +1,75 @@ +use ndarray::Array2; + +use crate::{errors::MessageNdarrayError, CodesError, KeyType, KeyedMessage}; + +impl KeyedMessage { + /// Returns [y, x] ([Nj, Ni], [lat, lon]) ndarray from the message, + /// x coordinates are increasing with the i index, + /// y coordinates are decreasing with the j index. + pub fn to_ndarray_2d(&self) -> Result, CodesError> { + let ni = if let KeyType::Int(ni) = self.read_key("Ni")?.value { + ni + } else { + return Err(MessageNdarrayError::UnexpectedKeyType("Ni".to_owned()).into()); + }; + + let nj = if let KeyType::Int(nj) = self.read_key("Nj")?.value { + nj + } else { + return Err(MessageNdarrayError::UnexpectedKeyType("Nj".to_owned()).into()); + }; + + let vals = if let KeyType::FloatArray(vals) = self.read_key("values")?.value { + vals + } else { + return Err(MessageNdarrayError::UnexpectedKeyType("values".to_owned()).into()); + }; + + if vals.len() != (ni * nj) as usize { + return Err(MessageNdarrayError::UnexpectedValuesLength(vals.len(), ni * nj).into()); + } + + let shape = (nj as usize, ni as usize); + let vals = Array2::from_shape_vec(shape, vals).map_err(|e| MessageNdarrayError::from(e))?; + + Ok(vals) + } +} + +#[cfg(test)] +mod tests { + use float_cmp::assert_approx_eq; + + use super::*; + use crate::codes_handle::CodesHandle; + use crate::FallibleStreamingIterator; + use crate::KeyType; + use crate::ProductKind; + use std::path::Path; + + #[test] + fn test_to_ndarray() -> Result<(), CodesError> { + let file_path = Path::new("./data/iceland-surface.grib"); + let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + + while let Some(msg) = handle.next()? { + if msg.read_key("shortName")?.value == KeyType::Str("2d".to_string()) { + let ndarray = msg.to_ndarray_2d()?; + + // values from xarray + assert_approx_eq!(f64, ndarray[[0, 0]], 276.37793, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[0, 48]], 276.65723, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[16, 0]], 277.91113, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[16, 48]], 280.34277, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[5, 5]], 276.03418, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[10, 10]], 277.59082, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[15, 15]], 277.68652, epsilon = 0.000_1); + assert_approx_eq!(f64, ndarray[[8, 37]], 273.2744, epsilon = 0.000_1); + + break; + } + } + + Ok(()) + } +} From f13125e5f4b130f221f137df2401bbac35fa2b80 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:05:25 +0100 Subject: [PATCH 3/7] fix index feature definitions --- src/intermediate_bindings/codes_handle.rs | 5 ++++- src/intermediate_bindings/mod.rs | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index f8c206f..18d95d9 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -1,6 +1,8 @@ use std::ptr::{self}; -use eccodes_sys::{codes_context, codes_handle, codes_index, CODES_LOCK}; +use eccodes_sys::{codes_context, codes_handle}; +#[cfg(feature = "experimental_index")] +use eccodes_sys::{codes_index, CODES_LOCK}; use libc::FILE; use num_traits::FromPrimitive; @@ -56,6 +58,7 @@ pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), Codes Ok(()) } +#[cfg(feature = "experimental_index")] pub unsafe fn codes_handle_new_from_index( index: *mut codes_index, ) -> Result<*mut codes_handle, CodesError> { diff --git a/src/intermediate_bindings/mod.rs b/src/intermediate_bindings/mod.rs index 23f49e2..a90e336 100644 --- a/src/intermediate_bindings/mod.rs +++ b/src/intermediate_bindings/mod.rs @@ -32,10 +32,10 @@ pub use codes_get::{ codes_get_long_array, codes_get_message, codes_get_native_type, codes_get_size, codes_get_string, }; -pub use codes_handle::{ - codes_handle_clone, codes_handle_delete, codes_handle_new_from_file, - codes_handle_new_from_index, -}; +#[cfg(feature = "experimental_index")] +pub use codes_handle::codes_handle_new_from_index; +pub use codes_handle::{codes_handle_clone, codes_handle_delete, codes_handle_new_from_file}; +#[cfg(feature = "experimental_index")] pub use codes_index::{ codes_index_add_file, codes_index_delete, codes_index_new, codes_index_read, codes_index_select_double, codes_index_select_long, codes_index_select_string, From 6cfcc63ffbc44a8e56b7d69e3cf775cae26ab0b0 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:09:18 +0100 Subject: [PATCH 4/7] add ndarray and fix gh actions --- .github/workflows/rust-dev.yml | 12 ++++++------ .github/workflows/rust.yml | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml index 590b270..f7b22f1 100644 --- a/.github/workflows/rust-dev.yml +++ b/.github/workflows/rust-dev.yml @@ -29,13 +29,13 @@ jobs: cargo clean - name: Test with cargo run: | - RUST_BACKTRACE=full cargo test --features "experimental_index" + RUST_BACKTRACE=full cargo test --features "experimental_index, message_ndarray" -- --include-ignored - name: Check with clippy run: | - cargo clippy --features "experimental_index" + cargo clippy --features "experimental_index, message_ndarray" - name: Build release run: | - cargo test --features "experimental_index" -- --include-ignored + cargo build --release --features "experimental_index, message_ndarray" build-macos: @@ -51,10 +51,10 @@ jobs: cargo clean - name: Test with cargo run: | - RUST_BACKTRACE=full cargo test --features "experimental_index" + RUST_BACKTRACE=full cargo test --features "experimental_index, message_ndarray" -- --include-ignored - name: Check with clippy run: | - cargo clippy --features "experimental_index" + cargo clippy --features "experimental_index, message_ndarray" - name: Build release run: | - cargo test --features "experimental_index" -- --include-ignored + cargo build --release --features "experimental_index, message_ndarray" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bb91cc0..c3c9113 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -27,11 +27,11 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release --features "experimental_index" + cargo build --release --features "experimental_index, message_ndarray" cargo clean - name: Test with cargo run: | - cargo test --features "experimental_index" + cargo test --features "experimental_index, message_ndarray" cargo clean - name: Benchmark with criterion run: | @@ -53,11 +53,11 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release --features "experimental_index" + cargo build --release --features "experimental_index, message_ndarray" cargo clean - name: Test with cargo run: | - cargo test --features "experimental_index" + cargo test --features "experimental_index, message_ndarray" cargo clean - name: Benchmark with criterion run: | From ff333deea7c6aadd2ed2a9cd3f7cc6ecc51c5dde Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:09:34 +0100 Subject: [PATCH 5/7] make ndarray default --- Cargo.toml | 1 + data/iceland-surface.grib.923a8.idx | Bin 0 -> 1354 bytes 2 files changed, 1 insertion(+) create mode 100755 data/iceland-surface.grib.923a8.idx diff --git a/Cargo.toml b/Cargo.toml index d47034f..b7ebf61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ anyhow = "1.0" float-cmp = "0.9" [features] +default = ["message_ndarray"] docs = ["eccodes-sys/docs"] experimental_index = [] message_ndarray = ["ndarray"] diff --git a/data/iceland-surface.grib.923a8.idx b/data/iceland-surface.grib.923a8.idx new file mode 100755 index 0000000000000000000000000000000000000000..5175d96bea555bcdd2cf89cea4b364a1e7edc7df GIT binary patch literal 1354 zcma)+&2AGh5XVzUn~%0>D+EFUp@>sTX)0BmxBw*-wV^3WQ*lCzvpXAG?ApQWEzO}) zIna}DHR1{QdH~*!v3HYzkfP?2S$pO;{~3FHUi|W8zTkd(#SaIP^_wvy2^m0QbY+)C zu*U;9HJ4`nlQ}aDT@D#UK?2Iqg05PJBPD^vQ;*`1C~EYb=2k!y*h+97cjD z6wGr@Ei(?o*Mf0{6_kcVl6Wt`S`tSs+rNxavF0&n)~}MJ{pWd2Mdbu9D$H~kzOso7 z=vv4k1=8z}vNoJqg|^)u3lh)u$F`=Si{Qth(c3RlDTY9JJ>%9xA?pWNXBuyI2_HcH z4d7XTtnUi!21!uH{xAR0T)UvJ5p3gpiaifa8yDz?u1XlB5s_XLnU~q6C=}X-gJ;*< z-zML>#CW0zS1xM*{Pg!bUCPhaD3sSOkPFkG4Z5DEq7UdsmVrK;U^49Y$v54!E)GYV z7IRQ8U7hoD24}T)bKeAEqkU$5pP=!f|b8hs;_ zBKGXiczEZ*a~J2*(5HG$iT~Fx484bYwP}>QlP+Ls9FQ+v7+pOf5eqV?u7oM~l@L*W zsTCs;jy2u+Y`S=&JL_)#oeG>%qnBJTQ=@zktu$NBUpsCt(~dHI+M%tC)3O}erAM?! zkKG;$shxmi;fV1;CI)N3;T%^U3yx2UD`I(EM3Bbu2-h%p*^S0Dw)2l<+SN;A2d9+w jrVFzKhA4253rzK`?0p(ap@c6Y4+SUgEt=cL=d1Y#d2HeG literal 0 HcmV?d00001 From 557b1357af3e7731a4cd6424aea7bf60096d29fa Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 4 Feb 2024 23:42:31 +0100 Subject: [PATCH 6/7] add reading ndarray with coordinates --- src/message_ndarray.rs | 70 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index a5b1ae5..d656967 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -1,4 +1,4 @@ -use ndarray::Array2; +use ndarray::{s, Array2, Array3}; use crate::{errors::MessageNdarrayError, CodesError, KeyType, KeyedMessage}; @@ -6,7 +6,7 @@ impl KeyedMessage { /// Returns [y, x] ([Nj, Ni], [lat, lon]) ndarray from the message, /// x coordinates are increasing with the i index, /// y coordinates are decreasing with the j index. - pub fn to_ndarray_2d(&self) -> Result, CodesError> { + pub fn to_ndarray(&self) -> Result, CodesError> { let ni = if let KeyType::Int(ni) = self.read_key("Ni")?.value { ni } else { @@ -34,6 +34,44 @@ impl KeyedMessage { Ok(vals) } + + pub fn to_lons_lats_values( + &self, + ) -> Result<(Array2, Array2, Array2), CodesError> { + let ni = if let KeyType::Int(ni) = self.read_key("Ni")?.value { + ni + } else { + return Err(MessageNdarrayError::UnexpectedKeyType("Ni".to_owned()).into()); + }; + + let nj = if let KeyType::Int(nj) = self.read_key("Nj")?.value { + nj + } else { + return Err(MessageNdarrayError::UnexpectedKeyType("Nj".to_owned()).into()); + }; + + let latlonvals = if let KeyType::FloatArray(vals) = self.read_key("latLonValues")?.value { + vals + } else { + return Err(MessageNdarrayError::UnexpectedKeyType("values".to_owned()).into()); + }; + + if latlonvals.len() != (ni * nj * 3) as usize { + return Err( + MessageNdarrayError::UnexpectedValuesLength(latlonvals.len(), ni * nj * 3).into(), + ); + } + + let shape = (nj as usize, ni as usize, 3_usize); + let mut latlonvals = + Array3::from_shape_vec(shape, latlonvals).map_err(|e| MessageNdarrayError::from(e))?; + let (lats, lons, vals) = + latlonvals + .view_mut() + .multi_slice_move((s![.., .., 0], s![.., .., 1], s![.., .., 2])); + + Ok((lons.into_owned(), lats.into_owned(), vals.into_owned())) + } } #[cfg(test)] @@ -54,7 +92,7 @@ mod tests { while let Some(msg) = handle.next()? { if msg.read_key("shortName")?.value == KeyType::Str("2d".to_string()) { - let ndarray = msg.to_ndarray_2d()?; + let ndarray = msg.to_ndarray()?; // values from xarray assert_approx_eq!(f64, ndarray[[0, 0]], 276.37793, epsilon = 0.000_1); @@ -72,4 +110,30 @@ mod tests { Ok(()) } + + #[test] + fn test_lons_lats() -> Result<(), CodesError> { + let file_path = Path::new("./data/iceland-surface.grib"); + let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + + while let Some(msg) = handle.next()? { + if msg.read_key("shortName")?.value == KeyType::Str("2d".to_string()) { + let (lats, lons, vals) = msg.to_lons_lats_values()?; + + // values from xarray + assert_approx_eq!(f64, vals[[0, 0]], 276.37793, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[0, 48]], 276.65723, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[16, 0]], 277.91113, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[16, 48]], 280.34277, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[5, 5]], 276.03418, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[10, 10]], 277.59082, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[15, 15]], 277.68652, epsilon = 0.000_1); + assert_approx_eq!(f64, vals[[8, 37]], 273.2744, epsilon = 0.000_1); + + break; + } + } + + todo!("Test lats and lons") + } } From a01e98d53763fa67b9971aa6e88aea416dde8832 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:26:17 +0100 Subject: [PATCH 7/7] extend ndarray test --- src/message_ndarray.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index d656967..1a1ae83 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -118,9 +118,9 @@ mod tests { while let Some(msg) = handle.next()? { if msg.read_key("shortName")?.value == KeyType::Str("2d".to_string()) { - let (lats, lons, vals) = msg.to_lons_lats_values()?; + let (lons, lats, vals) = msg.to_lons_lats_values()?; - // values from xarray + // values from cfgrib assert_approx_eq!(f64, vals[[0, 0]], 276.37793, epsilon = 0.000_1); assert_approx_eq!(f64, vals[[0, 48]], 276.65723, epsilon = 0.000_1); assert_approx_eq!(f64, vals[[16, 0]], 277.91113, epsilon = 0.000_1); @@ -130,10 +130,28 @@ mod tests { assert_approx_eq!(f64, vals[[15, 15]], 277.68652, epsilon = 0.000_1); assert_approx_eq!(f64, vals[[8, 37]], 273.2744, epsilon = 0.000_1); + assert_approx_eq!(f64, lons[[0, 0]], -25.0); + assert_approx_eq!(f64, lons[[0, 48]], -13.0); + assert_approx_eq!(f64, lons[[16, 0]], -25.0); + assert_approx_eq!(f64, lons[[16, 48]], -13.0); + assert_approx_eq!(f64, lons[[5, 5]], -23.75); + assert_approx_eq!(f64, lons[[10, 10]], -22.5); + assert_approx_eq!(f64, lons[[15, 15]], -21.25); + assert_approx_eq!(f64, lons[[8, 37]], -15.75); + + assert_approx_eq!(f64, lats[[0, 0]], 67.0); + assert_approx_eq!(f64, lats[[0, 48]], 67.0); + assert_approx_eq!(f64, lats[[16, 0]], 63.0); + assert_approx_eq!(f64, lats[[16, 48]], 63.0); + assert_approx_eq!(f64, lats[[5, 5]], 65.75); + assert_approx_eq!(f64, lats[[10, 10]], 64.5); + assert_approx_eq!(f64, lats[[15, 15]], 63.25); + assert_approx_eq!(f64, lats[[8, 37]], 65.0); + break; } } - todo!("Test lats and lons") + Ok(()) } }