chore(page): fix domain name clean handling

examples/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_examples"
-version = "1.34.3"
+version = "1.34.4"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "Multithreaded web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ htr = "0.5.27"
 flexbuffers = "2.0.0"
 
 [dependencies.spider]
-version = "1.34.3"
+version = "1.34.4"
 path = "../spider"
 features = ["serde"]
 

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "1.34.3"
+version = "1.34.4"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "The fastest web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"

spider/README.md

@@ -16,7 +16,7 @@ This is a basic async example crawling a web page, add spider to your `Cargo.tom
 
 ```toml
 [dependencies]
-spider = "1.34.3"
+spider = "1.34.4"
 ```
 
 And then the code:
@@ -87,7 +87,7 @@ We have a couple optional feature flags. Regex blacklisting, jemaloc backend, gl
 
 ```toml
 [dependencies]
-spider = { version = "1.34.3", features = ["regex", "ua_generator"] }
+spider = { version = "1.34.4", features = ["regex", "ua_generator"] }
 ```
 
 1. `ua_generator`: Enables auto generating a random real User-Agent.
@@ -108,7 +108,7 @@ Move processing to a worker, drastically increases performance even if worker is
 
 ```toml
 [dependencies]
-spider = { version = "1.34.3", features = ["decentralized"] }
+spider = { version = "1.34.4", features = ["decentralized"] }
 ```
 
 ```sh
@@ -128,7 +128,7 @@ Allow regex for blacklisting routes
 
 ```toml
 [dependencies]
-spider = { version = "1.34.3", features = ["regex"] }
+spider = { version = "1.34.4", features = ["regex"] }
 ```
 
 ```rust,no_run
@@ -155,7 +155,7 @@ If you are performing large workloads you may need to control the crawler by ena
 
 ```toml
 [dependencies]
-spider = { version = "1.34.3", features = ["control"] }
+spider = { version = "1.34.4", features = ["control"] }
 ```
 
 ```rust

spider/src/page.rs

@@ -50,12 +50,10 @@ pub fn domain_name(domain: &Url) -> &str {
         Some(b) => {
             let b = b.split('.').collect::<Vec<&str>>();
 
-            if b.len() > 2 {
-                b[1]
-            } else if b.len() == 2 {
-                b[0]
-            } else {
-                b[b.len() - 2]
+            if b.len() > 0 {
+                b[b.len() - 1]
+            }  else {
+                ""
             }
         }
         _ => "",

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "1.34.3"
+version = "1.34.4"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "The fastest web crawler CLI written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -26,7 +26,7 @@ quote = "1.0.18"
 failure_derive = "0.1.8"
 
 [dependencies.spider]
-version = "1.34.3"
+version = "1.34.4"
 path = "../spider"
 
 [[bin]]

spider_cli/README.md

@@ -40,7 +40,7 @@ spider --domain http://localhost:3000 download
 ```
 
 ```sh
-spider_cli 1.34.3
+spider_cli 1.34.4
 madeindjs <[email protected]>, j-mendez <[email protected]>
 The fastest web crawler CLI written in Rust.
 

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "1.34.3"
+version = "1.34.4"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "The fastest web crawler CLI written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ lazy_static = "1.4.0"
 env_logger = "0.10.0"
 
 [dependencies.spider]
-version = "1.34.3"
+version = "1.34.4"
 path = "../spider"
 features = ["serde", "flexbuffers"]