Brushing off the Dust (#24)

Some things have changed with the way Dune returns things: Response bytes can be bigger than u16 Floats are returned as strings Timestamps no longer have the T in the middle Most of these things are since the move from Spark to Trino (since this project has not been touched since then). Furthermore we Updated dependencies Fixed dead reference links Migrated an old query to Trino yada yada yada.. Nobody even watches this repo, none of this matters to anyone but me... But it will matter.
bh2smith · Nov 28, 2023 · b083c6d · b083c6d
1 parent ffdf6c1
commit b083c6d
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 25 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "duners"
-version = "0.0.1"
+version = "0.0.2"
 authors = ["Ben Smith <[email protected]>"]
 edition = "2021"
 description = "A simple framework for fetching query results from with [Dune Analytics API](https://dune.com/docs/api/)."
@@ -11,11 +11,11 @@ license = "MIT OR Apache-2.0"
 keywords = ["dune", "ethereum", "api-client", "web3", "dune-analytics"]
 
 [dependencies]
-chrono = { version ="0.4.23", features = ["serde", "rustc-serialize"] }
+chrono = { version = "0.4.31", features = ["serde"] }
 dotenv = "0.15.0"
-log = "0.4.17"
-reqwest = { version = "0.11.13", features = ["json"] }
-serde = { version = "1.0.151", features=["derive"] }
-serde_json = "1.0.91"
-serde_with = "2.1.0"
-tokio = { version = "1.23.0", features = ["full"] }
+log = "0.4.20"
+reqwest = { version = "0.11.22", features = ["json"] }
+serde = { version = "1.0.193", features = ["derive"] }
+serde_json = "1.0.108"
+serde_with = "3.4.0"
+tokio = { version = "1.34.0", features = ["full"] }
diff --git a/src/client.rs b/src/client.rs
@@ -94,7 +94,7 @@ impl DuneClient {
     }
 
     /// Execute Query (with or without parameters)
-    /// cf. [https://dune.com/docs/api/api-reference/execute-query-id](https://dune.com/docs/api/api-reference/execute-query-id)
+    /// cf. [https://dune.com/docs/api/api-reference/execute-queries/execute-query-id/](https://dune.com/docs/api/api-reference/execute-queries/execute-query-id/)
     pub async fn execute_query(
         &self,
         query_id: u32,
@@ -108,7 +108,7 @@ impl DuneClient {
     }
 
     /// Cancel Query Execution by `job_id`
-    /// cf. [https://dune.com/docs/api/api-reference/cancel-execution/](https://dune.com/docs/api/api-reference/cancel-execution/))
+    /// cf. [https://dune.com/docs/api/api-reference/execute-queries/cancel-execution/](https://dune.com/docs/api/api-reference/execute-queries/cancel-execution/)
     pub async fn cancel_execution(
         &self,
         job_id: &str,
@@ -121,7 +121,7 @@ impl DuneClient {
     }
 
     /// Get Query Execution Status (by `job_id`)
-    /// cf. [https://dune.com/docs/api/api-reference/execution-status/](https://dune.com/docs/api/api-reference/execution-status/)
+    /// cf. [https://dune.com/docs/api/api-reference/get-results/execution-status/](https://dune.com/docs/api/api-reference/get-results/execution-status/)
     pub async fn get_status(&self, job_id: &str) -> Result<GetStatusResponse, DuneRequestError> {
         let response = self
             ._get(job_id, "status")
@@ -131,7 +131,7 @@ impl DuneClient {
     }
 
     /// Get Query Execution Results (by `job_id`)
-    /// cf. [https://dune.com/docs/api/api-reference/execution-results/](https://dune.com/docs/api/api-reference/execution-results/)
+    /// cf. [https://dune.com/docs/api/api-reference/get-results/execution-results/](https://dune.com/docs/api/api-reference/get-results/execution-results/)
     pub async fn get_results<T: DeserializeOwned>(
         &self,
         job_id: &str,
@@ -151,7 +151,7 @@ impl DuneClient {
     /// * `query_id` - an integer representing query ID
     ///             (found at the end of a Dune Query URL: [https://dune.com/queries/971694](https://dune.com/queries/971694))
     /// * `parameters` - an optional list of query `Parameter`
-    ///             (cf. [https://dune.xyz/queries/1215383](https://dune.xyz/queries/1215383))
+    ///             (cf. [https://dune.xyz/queries/3238619](https://dune.xyz/queries/3238619))
     /// * `ping_frequency` - how frequently (in seconds) should the loop check execution status.
     ///             Default is 5 seconds. Too frequently could result in rate limiting
     ///             (i.e. Too Many Requests) especially when executing multiple queries in parallel.
@@ -160,7 +160,7 @@ impl DuneClient {
     /// ```
     /// use duners::{
     ///     client::DuneClient,
-    ///     dateutil::datetime_from_str,
+    ///     parse_utils::{datetime_from_str, f64_from_str},
     ///     error::DuneRequestError
     /// };
     /// use serde::Deserialize;
@@ -170,6 +170,7 @@ impl DuneClient {
     /// #[derive(Deserialize, Debug, PartialEq)]
     /// struct ResultStruct {
     ///     text_field: String,
+    ///     #[serde(deserialize_with = "f64_from_str")]
     ///     number_field: f64,
     ///     #[serde(deserialize_with = "datetime_from_str")]
     ///     date_field: DateTime<Utc>,
@@ -215,7 +216,7 @@ impl DuneClient {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::dateutil::{date_parse, datetime_from_str};
+    use crate::parse_utils::{date_parse, datetime_from_str, f64_from_str};
     use crate::response::ExecutionStatus;
     use chrono::{DateTime, Utc};
     use serde::Deserialize;
@@ -239,7 +240,7 @@ mod tests {
         let error = dune.execute_query(u32::MAX, None).await.unwrap_err();
         assert_eq!(
             error,
-            DuneRequestError::Dune(String::from("Query not found"))
+            DuneRequestError::Dune(String::from("An internal error occured"))
         )
     }
 
@@ -314,14 +315,15 @@ mod tests {
         #[derive(Deserialize, Debug, PartialEq)]
         struct ResultStruct {
             text_field: String,
+            #[serde(deserialize_with = "f64_from_str")]
             number_field: f64,
             #[serde(deserialize_with = "datetime_from_str")]
             date_field: DateTime<Utc>,
             list_field: String,
         }
         let results = dune
             .refresh::<ResultStruct>(
-                1215383,
+                3238619,
                 Some(vec![Parameter::number("NumberField", "3.141592653589793")]),
                 None,
             )

diff --git a/src/lib.rs b/src/lib.rs
@@ -1,10 +1,10 @@
 /// DuneClient structure and all API route implementations.
 pub mod client;
-/// Utility Methods (primarily for date parsing)
-pub mod dateutil;
 /// DuneRequestError (encapsulating all errors that could arise within network requests and result parsing)
 pub mod error;
 /// Content related to Query Parameters.
 pub mod parameters;
+/// Utility Methods (primarily for date parsing)
+pub mod parse_utils;
 /// Data models representing response types for all client methods.
 pub mod response;
diff --git a/src/parameters.rs b/src/parameters.rs
@@ -69,7 +69,7 @@ impl Parameter {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::dateutil::date_parse;
+    use crate::parse_utils::date_parse;
 
     #[test]
     fn new_parameter() {

diff --git a/src/dateutil.rs → src/parse_utils.rs b/src/dateutil.rs → src/parse_utils.rs
@@ -1,10 +1,11 @@
 #![allow(dead_code)]
 use chrono::{DateTime, NaiveDateTime, ParseError, Utc};
 use serde::{de, Deserialize, Deserializer};
+use serde_json::Value;
 
 fn date_string_parser(date_str: &str, format: &str) -> Result<DateTime<Utc>, ParseError> {
     let native = NaiveDateTime::parse_from_str(date_str, format);
-    Ok(DateTime::<Utc>::from_utc(native?, Utc))
+    Ok(DateTime::from_naive_utc_and_offset(native?, Utc))
 }
 
 /// The date format returned by DuneAPI response Date fields (e.g. `submitted_at`)
@@ -14,7 +15,7 @@ pub fn date_parse(date_str: &str) -> Result<DateTime<Utc>, ParseError> {
 
 /// The Date format returned from data fields of type timestamp.
 pub fn dune_date(date_str: &str) -> Result<DateTime<Utc>, ParseError> {
-    date_string_parser(date_str, "%Y-%m-%dT%H:%M:%S")
+    date_string_parser(date_str, "%Y-%m-%d %H:%M:%S.%f")
 }
 
 pub fn datetime_from_str<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
@@ -48,6 +49,18 @@ where
     }
 }
 
+pub fn f64_from_str<'de, D>(deserializer: D) -> Result<f64, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let value: Value = Deserialize::deserialize(deserializer)?;
+    if let Value::String(s) = value {
+        s.parse().map_err(serde::de::Error::custom)
+    } else {
+        Err(serde::de::Error::custom("Expected a string"))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -60,4 +73,13 @@ mod tests {
             "2022-01-01 01:02:03.000000123 UTC"
         )
     }
+
+    #[test]
+    fn new_dune_date() {
+        let date_str = "2022-05-04 00:00:00.000";
+        assert_eq!(
+            dune_date(date_str).unwrap().to_string(),
+            "2022-05-04 00:00:00 UTC"
+        )
+    }
 }
diff --git a/src/response.rs b/src/response.rs
@@ -1,4 +1,4 @@
-use crate::dateutil::{datetime_from_str, optional_datetime_from_str};
+use crate::parse_utils::{datetime_from_str, optional_datetime_from_str};
 use chrono::{DateTime, Utc};
 use serde::Deserialize;
 use serde_with::DeserializeFromStr;
@@ -66,7 +66,7 @@ pub struct CancellationResponse {
 #[derive(Deserialize, Debug)]
 pub struct ResultMetaData {
     pub column_names: Vec<String>,
-    pub result_set_bytes: u16,
+    pub result_set_bytes: u64,
     pub total_row_count: u32,
     pub datapoint_count: u32,
     pub pending_time_millis: Option<u32>,

diff --git a/tests/example.rs b/tests/example.rs
@@ -1,11 +1,15 @@
 use chrono::{DateTime, Utc};
-use duners::{client::DuneClient, dateutil::datetime_from_str};
+use duners::{
+    client::DuneClient, parameters::Parameter, parse_utils::datetime_from_str,
+    parse_utils::f64_from_str,
+};
 use serde::Deserialize;
 
 // User must declare the expected query return fields and types!
 #[derive(Deserialize, Debug, PartialEq)]
 struct ResultStruct {
     text_field: String,
+    #[serde(deserialize_with = "f64_from_str")]
     number_field: f64,
     #[serde(deserialize_with = "datetime_from_str")]
     date_field: DateTime<Utc>,
@@ -21,3 +25,46 @@ async fn test_external_use() {
         .unwrap();
     println!("{:?}", results.get_rows());
 }
+
+#[tokio::test]
+async fn test_blocks() {
+    #[allow(dead_code)]
+    #[derive(Deserialize, Debug, PartialEq)]
+    struct Block {
+        pub number: u64,
+        pub time: u64,
+    }
+
+    let dune = DuneClient::from_env();
+    let (start, end) = (5, 7);
+    let result = dune
+        .refresh::<Block>(
+            3238189,
+            Some(vec![
+                Parameter::number("Start", &start.to_string()),
+                Parameter::number("Width", &(end - start).to_string()),
+            ]),
+            Some(1),
+        )
+        .await
+        .unwrap();
+    println!("{:?}", result.execution_id);
+    println!("{:?}", result.result.metadata);
+    assert_eq!(
+        result.get_rows(),
+        vec![
+            Block {
+                number: 5,
+                time: 1438270083
+            },
+            Block {
+                number: 6,
+                time: 1438270107
+            },
+            Block {
+                number: 7,
+                time: 1438270110
+            }
+        ]
+    )
+}