From 921d8892bf8ea13e6bddb75dfcdc0d103fd22f65 Mon Sep 17 00:00:00 2001 From: j-mendez Date: Fri, 25 Oct 2024 15:58:35 -0400 Subject: [PATCH] chore(budget): fix whitelist/blacklist budgeting --- Cargo.lock | 12 ++++++------ spider/Cargo.toml | 2 +- spider/src/website.rs | 24 +++++++++++++----------- spider_chrome/Cargo.toml | 2 +- spider_cli/Cargo.toml | 2 +- spider_transformations/Cargo.toml | 2 +- spider_utils/Cargo.toml | 2 +- spider_worker/Cargo.toml | 2 +- 8 files changed, 25 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c15ec9e25..3593ca546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3909,7 +3909,7 @@ dependencies = [ [[package]] name = "spider" -version = "2.10.23" +version = "2.10.24" dependencies = [ "ahash", "async-openai", @@ -3970,7 +3970,7 @@ dependencies = [ [[package]] name = "spider_chrome" -version = "2.10.23" +version = "2.10.24" dependencies = [ "adblock", "async-tungstenite", @@ -4005,7 +4005,7 @@ dependencies = [ [[package]] name = "spider_cli" -version = "2.10.23" +version = "2.10.24" dependencies = [ "clap", "env_logger", @@ -4029,7 +4029,7 @@ dependencies = [ [[package]] name = "spider_transformations" -version = "2.10.23" +version = "2.10.24" dependencies = [ "aho-corasick", "fast_html2md", @@ -4051,7 +4051,7 @@ dependencies = [ [[package]] name = "spider_utils" -version = "2.10.23" +version = "2.10.24" dependencies = [ "indexmap 1.9.3", "serde", @@ -4063,7 +4063,7 @@ dependencies = [ [[package]] name = "spider_worker" -version = "2.10.23" +version = "2.10.24" dependencies = [ "env_logger", "lazy_static", diff --git a/spider/Cargo.toml b/spider/Cargo.toml index 4ae5ab24a..3e632cae2 100644 --- a/spider/Cargo.toml +++ b/spider/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider" -version = "2.10.23" +version = "2.10.24" authors = [ "j-mendez " ] diff --git a/spider/src/website.rs b/spider/src/website.rs index 27f41515b..f90f80d26 100644 --- a/spider/src/website.rs +++ b/spider/src/website.rs @@ -354,10 +354,14 @@ impl Website { pub fn is_allowed(&mut self, link: &CaseInsensitiveString) -> ProcessLinkStatus { if self.links_visited.contains(link) { ProcessLinkStatus::Blocked - } else if self.is_over_budget(link) { - ProcessLinkStatus::BudgetExceeded } else { - self.is_allowed_default(link.inner()) + let status = self.is_allowed_default(link.inner()); + + if status.eq(&ProcessLinkStatus::Allowed) && self.is_over_budget(link) { + ProcessLinkStatus::BudgetExceeded + } else { + status + } } } @@ -373,15 +377,13 @@ impl Website { pub fn is_allowed(&mut self, link: &CaseInsensitiveString) -> ProcessLinkStatus { if self.links_visited.contains(link) { ProcessLinkStatus::Blocked - } else if self.is_over_budget(&link) { - ProcessLinkStatus::BudgetExceeded - } else if self - .is_allowed_default(link) - .eq(&ProcessLinkStatus::Allowed) - { - ProcessLinkStatus::Allowed } else { - ProcessLinkStatus::Blocked + let status = self.is_allowed_default(link); + if status.eq(&ProcessLinkStatus::Allowed) && self.is_over_budget(&link) { + ProcessLinkStatus::BudgetExceeded + } else { + status + } } } diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml index d2b7284e9..2eb2ebeae 100644 --- a/spider_chrome/Cargo.toml +++ b/spider_chrome/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_chrome" -version = "2.10.23" +version = "2.10.24" rust-version = "1.70" authors = [ "j-mendez " diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml index de10bfb71..2e6f382c6 100644 --- a/spider_cli/Cargo.toml +++ b/spider_cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_cli" -version = "2.10.23" +version = "2.10.24" authors = [ "j-mendez " ] diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml index ddc5d4573..1b731cf6c 100644 --- a/spider_transformations/Cargo.toml +++ b/spider_transformations/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_transformations" -version = "2.10.23" +version = "2.10.24" authors = [ "j-mendez " ] diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml index 4b75e70fb..187c96cf6 100644 --- a/spider_utils/Cargo.toml +++ b/spider_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_utils" -version = "2.10.23" +version = "2.10.24" authors = [ "j-mendez " ] diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml index 817575037..b9f31963b 100644 --- a/spider_worker/Cargo.toml +++ b/spider_worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_worker" -version = "2.10.23" +version = "2.10.24" authors = [ "j-mendez " ]