diff --git a/Cargo.lock b/Cargo.lock index cc1ea9b6e7..a4563066f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,13 +5,11 @@ version = 3 [[package]] name = "ada-url" version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e5b772a9228e68a71f62764b5afe5a3ae34670a058ab9b844e72a38a3f01bc" dependencies = [ "cc", + "derive_more", "link_args", "serde", - "thiserror", ] [[package]] @@ -746,6 +744,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "cookie" version = "0.16.2" @@ -1097,6 +1101,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + [[package]] name = "diff" version = "0.1.13" @@ -3130,6 +3147,15 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.23" @@ -3279,6 +3305,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" + [[package]] name = "serde" version = "1.0.185" diff --git a/examples/collect_links/Cargo.toml b/examples/collect_links/Cargo.toml index a9d6e21359..14758ffcb5 100644 --- a/examples/collect_links/Cargo.toml +++ b/examples/collect_links/Cargo.toml @@ -9,7 +9,7 @@ path = "collect_links.rs" [dependencies] lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features = false } -ada-url = { version = "1.4.3" } +ada-url = { path = "../../../ada-rust", features = ["serde"] } tokio = { version = "1.32.0", features = ["full"] } regex = "1.9.3" http = "0.2.9" diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml index 52d0ef8942..40122c0ec9 100644 --- a/lychee-bin/Cargo.toml +++ b/lychee-bin/Cargo.toml @@ -19,7 +19,7 @@ version = "0.13.0" [dependencies] lychee-lib = { path = "../lychee-lib", version = "0.13.0", default-features = false } -ada-url = { version = "1.4.3", features = ["serde"] } +ada-url = { path = "../../ada-rust", features = ["serde"] } anyhow = "1.0.75" assert-json-diff = "2.0.2" clap = { version = "4.3.23", features = ["env", "derive"] } diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 5b1b176686..233c2355b8 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -298,7 +298,7 @@ async fn run(opts: &LycheeOptions) -> Result { .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").map_or(false, |x| x == "1")); if opts.config.dump_inputs { - let sources = collector.collect_sources(inputs).await; + let sources = collector.collect_sources(inputs); let exit_code = commands::dump_inputs(sources, opts.config.output.as_ref()).await?; return Ok(exit_code as i32); diff --git a/lychee-lib/Cargo.toml b/lychee-lib/Cargo.toml index c8574a75ff..ed85655c77 100644 --- a/lychee-lib/Cargo.toml +++ b/lychee-lib/Cargo.toml @@ -17,7 +17,7 @@ repository = "https://github.com/lycheeverse/lychee" version = "0.13.0" [dependencies] -ada-url = { version = "1.4.3", features = ["serde"] } +ada-url = { path = "../../ada-rust", features = ["serde"] } async-stream = "0.3.5" cached = "0.44.0" check-if-email-exists = { version = "0.9.0", optional = true } diff --git a/lychee-lib/src/collector.rs b/lychee-lib/src/collector.rs index 79d63b85c9..7c444a33bb 100644 --- a/lychee-lib/src/collector.rs +++ b/lychee-lib/src/collector.rs @@ -65,9 +65,9 @@ impl Collector { /// Collect all sources from a list of [`Input`]s. For further details, /// see also [`Input::get_sources`](crate::Input#method.get_sources). - pub async fn collect_sources(self, inputs: Vec) -> impl Stream> { + pub fn collect_sources(self, inputs: Vec) -> impl Stream> { stream::iter(inputs) - .par_then_unordered(None, move |input| async move { input.get_sources().await }) + .par_then_unordered(None, move |input| async move { input.get_sources() }) .flatten() } @@ -82,7 +82,7 @@ impl Collector { let skip_missing_inputs = self.skip_missing_inputs; let contents = stream::iter(inputs) .par_then_unordered(None, move |input| async move { - input.get_contents(skip_missing_inputs).await + input.get_contents(skip_missing_inputs) }) .flatten(); @@ -140,7 +140,7 @@ mod tests { let file_path = temp_dir.path().join("README"); let _file = File::create(&file_path).unwrap(); let input = Input::new(&file_path.as_path().display().to_string(), None, true, None)?; - let contents: Vec<_> = input.get_contents(true).await.collect::>().await; + let contents: Vec<_> = input.get_contents(true).collect::>().await; assert_eq!(contents.len(), 1); assert_eq!(contents[0].as_ref().unwrap().file_type, FileType::Plaintext); @@ -150,7 +150,7 @@ mod tests { #[tokio::test] async fn test_url_without_extension_is_html() -> Result<()> { let input = Input::new("https://example.com/", None, true, None)?; - let contents: Vec<_> = input.get_contents(true).await.collect::>().await; + let contents: Vec<_> = input.get_contents(true).collect::>().await; assert_eq!(contents.len(), 1); assert_eq!(contents[0].as_ref().unwrap().file_type, FileType::Html); diff --git a/lychee-lib/src/test_utils.rs b/lychee-lib/src/test_utils.rs index 97010efae1..321d8303f5 100644 --- a/lychee-lib/src/test_utils.rs +++ b/lychee-lib/src/test_utils.rs @@ -41,10 +41,11 @@ pub(crate) fn website(url: &str) -> Uri { /// Creates a mail URI from a string pub(crate) fn mail(address: &str) -> Uri { + let input = "mailto:".to_string() + address; if address.starts_with("mailto:") { Url::parse(address, None) } else { - Url::parse(&("mailto:".to_string() + address), None) + Url::parse(input.as_str(), None) } .expect("Expected valid Mail Address") .into() diff --git a/lychee-lib/src/types/input.rs b/lychee-lib/src/types/input.rs index fc65f83f01..3919d04e4a 100644 --- a/lychee-lib/src/types/input.rs +++ b/lychee-lib/src/types/input.rs @@ -180,10 +180,7 @@ impl Input { /// Returns an error if the contents can not be retrieved /// because of an underlying I/O error (e.g. an error while making a /// network request or retrieving the contents from the file system) - pub async fn get_contents( - self, - skip_missing: bool, - ) -> impl Stream> { + pub fn get_contents(self, skip_missing: bool) -> impl Stream> { try_stream! { match self.source { InputSource::RemoteUrl(ref url) => { @@ -198,7 +195,7 @@ impl Input { ref pattern, ignore_case, } => { - for await content in self.glob_contents(pattern, ignore_case).await { + for await content in self.glob_contents(pattern, ignore_case) { let content = content?; yield content; } @@ -271,7 +268,7 @@ impl Input { /// # Errors /// /// Returns an error if the globbing fails with the expanded pattern. - pub async fn get_sources(self) -> impl Stream> { + pub fn get_sources(self) -> impl Stream> { try_stream! { match self.source { InputSource::RemoteUrl(url) => yield url.to_string(), @@ -315,7 +312,7 @@ impl Input { Ok(input_content) } - async fn glob_contents( + fn glob_contents( &self, pattern: &str, ignore_case: bool, diff --git a/lychee-lib/src/types/uri/valid.rs b/lychee-lib/src/types/uri/valid.rs index 485df0d9fb..ba46dfb6ee 100644 --- a/lychee-lib/src/types/uri/valid.rs +++ b/lychee-lib/src/types/uri/valid.rs @@ -43,7 +43,7 @@ impl Uri { #[inline] /// Changes this URL's scheme. pub(crate) fn set_scheme(&mut self, scheme: &str) { - self.url.set_protocol(scheme); + let _ = self.url.set_protocol(scheme); } #[inline] diff --git a/lychee-lib/src/utils/fragment_checker.rs b/lychee-lib/src/utils/fragment_checker.rs index d8f6208fb4..d0436453de 100644 --- a/lychee-lib/src/utils/fragment_checker.rs +++ b/lychee-lib/src/utils/fragment_checker.rs @@ -65,7 +65,7 @@ impl FragmentChecker { } fn remove_fragment(mut url: Url) -> String { - url.set_hash(""); + url.set_hash(None); url.into() } } diff --git a/lychee-lib/src/utils/request.rs b/lychee-lib/src/utils/request.rs index 20879ad528..b13010f5d3 100644 --- a/lychee-lib/src/utils/request.rs +++ b/lychee-lib/src/utils/request.rs @@ -138,11 +138,7 @@ fn construct_url(base: &Option, text: &str) -> Result { }) } -fn create_uri_from_path( - src: &Path, - dst: &str, - base: &Option, -) -> Result> { +fn create_uri_from_path(src: &Path, dst: &str, base: &Option) -> Result> { let (dst, frag) = url::remove_get_params_and_separate_fragment(dst); // Avoid double-encoding already encoded destination paths by removing any // potential encoding (e.g. `web%20site` becomes `web site`). @@ -156,13 +152,12 @@ fn create_uri_from_path( let decoded = percent_decode_str(dst).decode_utf8()?; let resolved = path::resolve(src, &PathBuf::from(&*decoded), base)?; match resolved { - Some(path) => ada_url::Url::parse(path.to_str().unwrap(), Some("file://")) + Some(path) => Url::parse(path.to_str().unwrap(), Some("file://")) .map(|mut url| { - url.set_hash(frag.unwrap_or("")); - url + url.set_hash(frag); + Some(url) }) - .map(Some) - .map_err(|_e| ErrorKind::InvalidUrlFromPath(path)), + .map_err(|_e| ErrorKind::InvalidUrlFromPath(path.clone())), None => Ok(None), } }