From a502d82d00719e9670938ad710cf0443f7c81f4d Mon Sep 17 00:00:00 2001
From: Mateusz Jasiuk <mateusz.jasiuk@gmail.com>
Date: Fri, 13 Sep 2024 18:36:41 +0200
Subject: [PATCH] feat: reduce chain crawler interval (#118)

* feat: reduce crawlers interval

* feat: reduce chain crawler interval time
---
 chain/src/main.rs        | 1 +
 governance/src/main.rs   | 1 +
 parameters/src/main.rs   | 1 +
 pos/src/main.rs          | 1 +
 rewards/src/main.rs      | 1 +
 shared/src/crawler.rs    | 9 +++++++--
 transactions/src/main.rs | 1 +
 7 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/chain/src/main.rs b/chain/src/main.rs
index cadadac7..3970d7ef 100644
--- a/chain/src/main.rs
+++ b/chain/src/main.rs
@@ -90,6 +90,7 @@ async fn main() -> Result<(), MainError> {
             )
         },
         crawler_state.last_processed_block,
+        Some(1000),
     )
     .await
 }
diff --git a/governance/src/main.rs b/governance/src/main.rs
index 771c8a0b..3f1d72be 100644
--- a/governance/src/main.rs
+++ b/governance/src/main.rs
@@ -73,6 +73,7 @@ async fn main() -> Result<(), MainError> {
             )
         },
         0,
+        None,
     )
     .await
 }
diff --git a/parameters/src/main.rs b/parameters/src/main.rs
index 4605a11b..8b1183e9 100644
--- a/parameters/src/main.rs
+++ b/parameters/src/main.rs
@@ -74,6 +74,7 @@ async fn main() -> Result<(), MainError> {
             )
         },
         0,
+        None,
     )
     .await
 }
diff --git a/pos/src/main.rs b/pos/src/main.rs
index acae4ad4..807003a4 100644
--- a/pos/src/main.rs
+++ b/pos/src/main.rs
@@ -58,6 +58,7 @@ async fn main() -> Result<(), MainError> {
     crawler::crawl(
         move |epoch| crawling_fn(epoch, conn.clone(), client.clone()),
         next_epoch,
+        None,
     )
     .await
 }
diff --git a/rewards/src/main.rs b/rewards/src/main.rs
index d54b887b..40bf7076 100644
--- a/rewards/src/main.rs
+++ b/rewards/src/main.rs
@@ -73,6 +73,7 @@ async fn main() -> Result<(), MainError> {
     crawler::crawl(
         move |epoch| crawling_fn(conn.clone(), client.clone(), epoch),
         epoch,
+        None,
     )
     .await
 }
diff --git a/shared/src/crawler.rs b/shared/src/crawler.rs
index 84d5a65a..beddd92b 100644
--- a/shared/src/crawler.rs
+++ b/shared/src/crawler.rs
@@ -20,14 +20,19 @@ fn indexes(from: u32, to: Option<u32>) -> impl Stream<Item = u32> {
     }
 }
 
-pub async fn crawl<F, Fut>(f: F, first_index: u32) -> Result<(), MainError>
+pub async fn crawl<F, Fut>(
+    f: F,
+    first_index: u32,
+    interval: Option<u64>,
+) -> Result<(), MainError>
 where
     F: Fn(u32) -> Fut,
     Fut: Future<Output = Result<(), MainError>>,
 {
+    let interval = interval.unwrap_or(5000);
     let s = indexes(first_index, None);
     pin_mut!(s);
-    let retry_strategy = FixedInterval::from_millis(5000).map(jitter);
+    let retry_strategy = FixedInterval::from_millis(interval).map(jitter);
     let must_exit = must_exit_handle();
 
     while let Some(index) = s.next().await {
diff --git a/transactions/src/main.rs b/transactions/src/main.rs
index 50b142cc..dabadbe6 100644
--- a/transactions/src/main.rs
+++ b/transactions/src/main.rs
@@ -83,6 +83,7 @@ async fn main() -> Result<(), MainError> {
             )
         },
         next_block,
+        None,
     )
     .await
 }