Skip to content

Commit

Permalink
chore(smart): fix establishing whitelist/blacklist
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mendez committed Oct 26, 2024
1 parent 921d889 commit 6294ab5
Show file tree
Hide file tree
Showing 10 changed files with 103 additions and 109 deletions.
16 changes: 8 additions & 8 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion spider/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider"
version = "2.10.24"
version = "2.10.25"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
41 changes: 41 additions & 0 deletions spider/src/features/chrome.rs
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,47 @@ pub async fn setup_chrome_interception_base(
None
}

/// establish all the page events.
pub async fn setup_chrome_events(chrome_page: &chromiumoxide::Page, config: &Configuration) {
let stealth = async {
if cfg!(feature = "chrome_stealth") || config.stealth_mode {
match config.user_agent.as_ref() {
Some(agent) => {
let _ = chrome_page.enable_stealth_mode_with_agent(agent).await;
}
_ => {
let _ = chrome_page.enable_stealth_mode().await;
}
}
}
};
let eval_docs = async {
match config.evaluate_on_new_document {
Some(ref script) => {
if config.fingerprint {
let _ = chrome_page
.evaluate_on_new_document(string_concat!(
crate::features::chrome::FP_JS,
script.as_str()
))
.await;
} else {
let _ = chrome_page.evaluate_on_new_document(script.as_str()).await;
}
}
_ => {
if config.fingerprint {
let _ = chrome_page
.evaluate_on_new_document(crate::features::chrome::FP_JS)
.await;
}
}
}
};

tokio::join!(stealth, eval_docs, configure_browser(&chrome_page, &config));
}

/// static chrome arguments to start
#[cfg(all(feature = "chrome_cpu", feature = "real_browser"))]
pub static CHROME_ARGS: [&'static str; 27] = [
Expand Down
2 changes: 1 addition & 1 deletion spider/src/page.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1164,7 +1164,7 @@ impl Page {
)
.await;

crate::website::Website::setup_chrome_events(&new_page, &configuration).await;
crate::website:: crate::features::chrome::setup_chrome_events(&new_page, &configuration).await;

let page_resource =
crate::utils::fetch_page_html_chrome_base(
Expand Down
Loading

0 comments on commit 6294ab5

Please sign in to comment.