chore(transformations): add filter svg markdown

spider-rs · Oct 20, 2024 · b3e1aea · b3e1aea
1 parent 7963e1e
commit b3e1aea
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 12 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.9.10"
+version = "2.9.12"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.9.10"
+version = "2.9.12"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.9.10"
+version = "2.9.12"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.9.10"
+version = "2.9.12"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/src/transformation/content.rs b/spider_transformations/src/transformation/content.rs
@@ -102,6 +102,8 @@ pub struct TransformConfig {
     pub return_format: ReturnFormat,
     /// Filter Images.
     pub filter_images: bool,
+    /// Trim the content for LLMs.
+    pub clean_html: bool
 }
 
 /// ignore tags for markdown transformation
@@ -332,15 +334,20 @@ pub fn transform_content(
             tag_factory.insert(String::from("noscript"), tag.clone());
 
             if filter_images {
+                tag_factory.insert(String::from("svg"), tag.clone());
                 tag_factory.insert(String::from("img"), tag.clone());
                 tag_factory.insert(String::from("picture"), tag.clone());
+            }
+
+            if c.clean_html {
                 html = clean_html(&html)
             }
 
             tag_factory.insert(String::from("iframe"), tag);
 
             let html = html2md::parse_html_custom(&html, &tag_factory, false);
             let html = aho_clean_markdown(&html);
+
             html
         }
         ReturnFormat::Html2Text => match encoding {

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.9.10"
+version = "2.9.12"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.9.10"
+version = "2.9.12"
 authors = [
     "j-mendez <[email protected]>"
 ]