From bb0551dcccd40f082ed762693c1a5a86d3e17fbe Mon Sep 17 00:00:00 2001 From: j-mendez Date: Thu, 24 Oct 2024 10:12:24 -0400 Subject: [PATCH] chore(smart): add layui.js and jquery detection --- Cargo.lock | 12 ++++++------ spider/Cargo.toml | 2 +- spider/src/page.rs | 9 +++++---- spider_chrome/Cargo.toml | 2 +- spider_cli/Cargo.toml | 2 +- spider_transformations/Cargo.toml | 2 +- spider_utils/Cargo.toml | 2 +- spider_worker/Cargo.toml | 2 +- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9110a25d..ce20cd6fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3894,7 +3894,7 @@ dependencies = [ [[package]] name = "spider" -version = "2.10.19" +version = "2.10.20" dependencies = [ "ahash", "async-openai", @@ -3955,7 +3955,7 @@ dependencies = [ [[package]] name = "spider_chrome" -version = "2.10.19" +version = "2.10.20" dependencies = [ "adblock", "async-tungstenite", @@ -3990,7 +3990,7 @@ dependencies = [ [[package]] name = "spider_cli" -version = "2.10.19" +version = "2.10.20" dependencies = [ "clap", "env_logger", @@ -4014,7 +4014,7 @@ dependencies = [ [[package]] name = "spider_transformations" -version = "2.10.19" +version = "2.10.20" dependencies = [ "aho-corasick", "fast_html2md", @@ -4035,7 +4035,7 @@ dependencies = [ [[package]] name = "spider_utils" -version = "2.10.19" +version = "2.10.20" dependencies = [ "indexmap 1.9.3", "serde", @@ -4047,7 +4047,7 @@ dependencies = [ [[package]] name = "spider_worker" -version = "2.10.19" +version = "2.10.20" dependencies = [ "env_logger", "lazy_static", diff --git a/spider/Cargo.toml b/spider/Cargo.toml index b03284b28..62e5ad2fe 100644 --- a/spider/Cargo.toml +++ b/spider/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider" -version = "2.10.19" +version = "2.10.20" authors = [ "j-mendez " ] diff --git a/spider/src/page.rs b/spider/src/page.rs index 3fb929637..7dbe02634 100644 --- a/spider/src/page.rs +++ b/spider/src/page.rs @@ -35,8 +35,8 @@ lazy_static! { pub static ref JS_FRAMEWORK_ASSETS: phf::Set<&'static str> = { phf::phf_set! { "jquery.min.js", "jquery.qtip.min.js", "jquery.js", "angular.js", "jquery.slim.js", "react.development.js", "react-dom.development.js", "react.production.min.js", "react-dom.production.min.js", - "vue.global.js", "vue.esm-browser.js", "vue.js", "bootstrap.min.js", "bootstrap.bundle.min.js", "bootstrap.esm.min.js", "d3.min.js", "d3.js", "material-components-web.min.js", - "otSDKStub.js", "clipboard.min.js", "moment.js", "moment.min.js", "dexie.js" + "vue.global.js", "vue.global.prod.js", "vue.esm-browser.js", "vue.js", "bootstrap.min.js", "bootstrap.bundle.min.js", "bootstrap.esm.min.js", "d3.min.js", "d3.js", "material-components-web.min.js", + "otSDKStub.js", "clipboard.min.js", "moment.js", "moment.min.js", "dexie.js", "layui.js" } }; } @@ -49,9 +49,9 @@ lazy_static! { // Add allowed assets from JS_FRAMEWORK_ASSETS except the excluded ones "jquery.min.js", "jquery.qtip.min.js", "jquery.js", "angular.js", "jquery.slim.js", "react.development.js", "react-dom.development.js", "react.production.min.js", - "react-dom.production.min.js", "vue.global.js", "vue.esm-browser.js", "vue.js", + "react-dom.production.min.js", "vue.global.js", "vue.global.prod.js", "vue.esm-browser.js", "vue.js", "bootstrap.min.js", "bootstrap.bundle.min.js", "bootstrap.esm.min.js", "d3.min.js", - "d3.js", + "d3.js", "layui.js", // Verified 3rd parties for request "https://m.stripe.network/inner.html", "https://m.stripe.network/out-4.5.43.js", @@ -1121,6 +1121,7 @@ impl Page { r"/.append/gm", r"/.appendChild/gm", r"/.write/gm", + r"\$\s*\(.*?\)", ]) .unwrap_unchecked() }; diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml index d3acf8736..28e008773 100644 --- a/spider_chrome/Cargo.toml +++ b/spider_chrome/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_chrome" -version = "2.10.19" +version = "2.10.20" rust-version = "1.70" authors = [ "j-mendez " diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml index ff77bfe97..416ddc9e1 100644 --- a/spider_cli/Cargo.toml +++ b/spider_cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_cli" -version = "2.10.19" +version = "2.10.20" authors = [ "j-mendez " ] diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml index a94ae610c..b43a84e2d 100644 --- a/spider_transformations/Cargo.toml +++ b/spider_transformations/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_transformations" -version = "2.10.19" +version = "2.10.20" authors = [ "j-mendez " ] diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml index e5e3b47ee..79d813184 100644 --- a/spider_utils/Cargo.toml +++ b/spider_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_utils" -version = "2.10.19" +version = "2.10.20" authors = [ "j-mendez " ] diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml index c2672f143..9b8e28ae5 100644 --- a/spider_worker/Cargo.toml +++ b/spider_worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_worker" -version = "2.10.19" +version = "2.10.20" authors = [ "j-mendez " ]