chore(chrome): add waf mouse movement

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.8.4"
+version = "2.8.5"
 authors = [
     "j-mendez <[email protected]>"
 ]
@@ -64,6 +64,7 @@ auto_encoder = { version = "0.1", optional = true }
 base64 = { version = "0.22", optional = true }
 string-interner = {version = "0.17", default-features = false,  features = ["std", "inline-more", "backends"]}
 httpdate = { version = "1", optional = true }
+rand = { version = "0.8", optional = true }
 
 [target.'cfg(all(not(windows), not(target_os = "android"), not(target_env = "musl")))'.dependencies]
 tikv-jemallocator = { version = "0.6", optional = true }
@@ -139,7 +140,7 @@ chrome_intercept = ["chrome"]
 chrome_headless_new = ["chrome"]
 cookies = ["reqwest/cookies"]
 cron = ["dep:async_job", "dep:chrono", "dep:cron", "dep:async-trait"]
-smart = ["chrome", "dep:regex", "chrome_intercept"]
+smart = ["chrome", "dep:regex", "dep:rand", "chrome_intercept"]
 encoding = ["dep:auto_encoder"]
 headers = ["dep:httpdate"]
 real_browser = []

spider/src/utils/mod.rs

@@ -833,6 +833,25 @@ async fn navigate(
     Ok(())
 }
 
+#[cfg(all(feature = "real_browser", feature = "smart"))]
+/// generate random mouse movement.
+async fn perform_smart_mouse_movement(page: &chromiumoxide::Page) {
+    use chromiumoxide::layout::Point;
+    use rand::rngs::SmallRng;
+    use rand::{Rng, SeedableRng};
+
+    let mut rng = SmallRng::from_entropy();
+
+    // we can pass in the browser size once we allow re-adjusting it and real movements.
+    let random_x = rng.gen_range(0.0..1280.0);
+    let random_y = rng.gen_range(0.0..720.0);
+
+    let _ = page.move_mouse(Point::new(random_x, random_y)).await;
+}
+
+#[cfg(all(not(feature = "real_browser"), feature = "smart"))]
+async fn perform_smart_mouse_movement(_page: &chromiumoxide::Page) {}
+
 #[cfg(feature = "chrome")]
 /// Perform a network request to a resource extracting all content as text streaming via chrome.
 pub async fn fetch_page_html_chrome_base(
@@ -859,6 +878,10 @@ pub async fn fetch_page_html_chrome_base(
         }
     }
 
+    if chrome_http_req_res.waf_check {
+        perform_smart_mouse_movement(&page).await;
+    }
+
     // we do not need to wait for navigation if content is assigned. The method set_content already handles this.
     let final_url = if wait_for_navigation && !content {
         let last_redirect = tokio::time::timeout(tokio::time::Duration::from_secs(15), async {

spider_chrome/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.8.4"
+version = "2.8.5"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"
@@ -48,7 +48,7 @@ reqwest = { version = "0.12", default-features = false }
 lazy_static = "1.5.0"
 phf = { version = "0.11", features = ["macros"] }
 adblock = { version = "0.8", optional = true, default-features = false, features = ["embedded-domain-resolver", "full-regex-handling"] }
-rand = "0.8.5"
+rand = "0.8"
 
 [target.'cfg(windows)'.dependencies]
 winreg = "0.52"

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.8.4"
+version = "2.8.5"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_transformations/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.8.4"
+version = "2.8.5"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.8.4"
+version = "2.8.5"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.8.4"
+version = "2.8.5"
 authors = [
     "j-mendez <[email protected]>"
 ]