chore(status): add rate limit and server status tracking

spider-rs · Oct 7, 2024 · 81fe9cd · 81fe9cd
1 parent 0ad5800
commit 81fe9cd
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 17 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.8.16"
+version = "2.8.17"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -170,6 +170,10 @@ pub enum CrawlStatus {
     Active,
     /// The crawl blocked from network ratelimit, firewall, etc.
     Blocked,
+    /// The crawl failed from a server error.
+    ServerError,
+    /// The crawl was rate limited.
+    RateLimited,
     /// The initial request ran without returning html.
     Empty,
     /// The URL of the website is invalid. Crawl cannot commence.
@@ -1323,8 +1327,11 @@ impl Website {
 
             if page.status_code == reqwest::StatusCode::FORBIDDEN && links.len() == 0 {
                 self.status = CrawlStatus::Blocked;
+            } else if page.status_code == reqwest::StatusCode::TOO_MANY_REQUESTS {
+                self.status = CrawlStatus::RateLimited;
+            } else if page.status_code.is_server_error() {
+                self.status = CrawlStatus::ServerError;
             }
-
             if self.configuration.return_page_links {
                 page.page_links = if links.is_empty() {
                     None
@@ -1473,8 +1480,11 @@ impl Website {
 
             if page.status_code == reqwest::StatusCode::FORBIDDEN && links.len() == 0 {
                 self.status = CrawlStatus::Blocked;
+            } else if page.status_code == reqwest::StatusCode::TOO_MANY_REQUESTS {
+                self.status = CrawlStatus::RateLimited;
+            } else if page.status_code.is_server_error() {
+                self.status = CrawlStatus::ServerError;
             }
-
             if self.configuration.return_page_links {
                 page.page_links = if links.is_empty() {
                     None
@@ -1655,8 +1665,11 @@ impl Website {
 
             if page.status_code == reqwest::StatusCode::FORBIDDEN && links.len() == 0 {
                 self.status = CrawlStatus::Blocked;
+            } else if page.status_code == reqwest::StatusCode::TOO_MANY_REQUESTS {
+                self.status = CrawlStatus::RateLimited;
+            } else if page.status_code.is_server_error() {
+                self.status = CrawlStatus::ServerError;
             }
-
             if self.configuration.return_page_links {
                 page.page_links = if links.is_empty() {
                     None
@@ -1709,10 +1722,13 @@ impl Website {
                 _ => *self.url.to_owned(),
             });
 
-            if page.status_code == reqwest::StatusCode::FORBIDDEN && page.links.len() == 0 {
+            if page.status_code == reqwest::StatusCode::FORBIDDEN && links.len() == 0 {
                 self.status = CrawlStatus::Blocked;
+            } else if page.status_code == reqwest::StatusCode::TOO_MANY_REQUESTS {
+                self.status = CrawlStatus::RateLimited;
+            } else if page.status_code.is_server_error() {
+                self.status = CrawlStatus::ServerError;
             }
-
             let links = HashSet::from(page.links.clone());
 
             if self.configuration.return_page_links {

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.8.16"
+version = "2.8.17"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.8.16"
+version = "2.8.17"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.8.16"
+version = "2.8.17"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.8.16"
+version = "2.8.17"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.8.16"
+version = "2.8.17"
 authors = [
     "j-mendez <[email protected]>"
 ]