chore(budget): fix whitelist/blacklist budgeting

spider-rs · Oct 25, 2024 · 921d889 · 921d889
1 parent fe995e8
commit 921d889
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 23 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.10.23"
+version = "2.10.24"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -354,10 +354,14 @@ impl Website {
     pub fn is_allowed(&mut self, link: &CaseInsensitiveString) -> ProcessLinkStatus {
         if self.links_visited.contains(link) {
             ProcessLinkStatus::Blocked
-        } else if self.is_over_budget(link) {
-            ProcessLinkStatus::BudgetExceeded
         } else {
-            self.is_allowed_default(link.inner())
+            let status = self.is_allowed_default(link.inner());
+
+            if status.eq(&ProcessLinkStatus::Allowed) && self.is_over_budget(link) {
+                ProcessLinkStatus::BudgetExceeded
+            } else {
+                status
+            }
         }
     }
 
@@ -373,15 +377,13 @@ impl Website {
     pub fn is_allowed(&mut self, link: &CaseInsensitiveString) -> ProcessLinkStatus {
         if self.links_visited.contains(link) {
             ProcessLinkStatus::Blocked
-        } else if self.is_over_budget(&link) {
-            ProcessLinkStatus::BudgetExceeded
-        } else if self
-            .is_allowed_default(link)
-            .eq(&ProcessLinkStatus::Allowed)
-        {
-            ProcessLinkStatus::Allowed
         } else {
-            ProcessLinkStatus::Blocked
+            let status = self.is_allowed_default(link);
+            if status.eq(&ProcessLinkStatus::Allowed) && self.is_over_budget(&link) {
+                ProcessLinkStatus::BudgetExceeded
+            } else {
+                status
+            }
         }
     }
 

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.10.23"
+version = "2.10.24"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.10.23"
+version = "2.10.24"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.10.23"
+version = "2.10.24"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.10.23"
+version = "2.10.24"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.10.23"
+version = "2.10.24"
 authors = [
     "j-mendez <[email protected]>"
 ]