From c2fed4c148070adecf86f73facb3613f6da3730e Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Tue, 26 Mar 2024 15:32:12 -0400 Subject: [PATCH] Read package metadata from pyproject.toml --- Cargo.lock | 2 + crates/pep508-rs/src/lib.rs | 54 ++++++- crates/pypi-types/Cargo.toml | 2 + crates/pypi-types/src/metadata.rs | 183 +++++++++++++++++++++-- crates/uv-distribution/src/error.rs | 6 +- crates/uv-distribution/src/source/mod.rs | 48 +++++- 6 files changed, 278 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e17c8264318fe..78d6d1a897328 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2750,6 +2750,7 @@ name = "pypi-types" version = "0.0.1" dependencies = [ "chrono", + "indexmap", "mailparse", "once_cell", "pep440_rs", @@ -2758,6 +2759,7 @@ dependencies = [ "rkyv", "serde", "thiserror", + "toml", "tracing", "url", "uv-normalize", diff --git a/crates/pep508-rs/src/lib.rs b/crates/pep508-rs/src/lib.rs index 1aa88cd340562..d37eff711dc95 100644 --- a/crates/pep508-rs/src/lib.rs +++ b/crates/pep508-rs/src/lib.rs @@ -471,6 +471,32 @@ impl Requirement { (true, Vec::new()) } } + + /// Return the requirement with an additional marker added, to require the given extra. + /// + /// For example, given `flask >= 2.0.2`, calling `with_extra_marker("dotenv")` would return + /// `flask >= 2.0.2 ; extra == "dotenv"`. + pub fn with_extra_marker(self, extra: &ExtraName) -> Self { + let marker = match self.marker { + Some(expression) => MarkerTree::And(vec![ + expression, + MarkerTree::Expression(MarkerExpression { + l_value: MarkerValue::Extra, + operator: MarkerOperator::Equal, + r_value: MarkerValue::QuotedString(extra.to_string()), + }), + ]), + None => MarkerTree::Expression(MarkerExpression { + l_value: MarkerValue::Extra, + operator: MarkerOperator::Equal, + r_value: MarkerValue::QuotedString(extra.to_string()), + }), + }; + Self { + marker: Some(marker), + ..self + } + } } impl UnnamedRequirement { @@ -1560,7 +1586,7 @@ mod tests { use insta::assert_snapshot; use pep440_rs::{Operator, Version, VersionPattern, VersionSpecifier}; - use uv_normalize::{ExtraName, PackageName}; + use uv_normalize::{ExtraName, InvalidNameError, PackageName}; use crate::marker::{ parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, @@ -2264,4 +2290,30 @@ mod tests { Ok(()) } + + #[test] + fn add_extra_marker() -> Result<(), InvalidNameError> { + let requirement = Requirement::from_str("pytest").unwrap(); + let expected = Requirement::from_str("pytest; extra == 'dotenv'").unwrap(); + let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?); + assert_eq!(actual, expected); + + let requirement = Requirement::from_str("pytest; '4.0' >= python_version").unwrap(); + let expected = + Requirement::from_str("pytest; '4.0' >= python_version and extra == 'dotenv'").unwrap(); + let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?); + assert_eq!(actual, expected); + + let requirement = + Requirement::from_str("pytest; '4.0' >= python_version or sys_platform == 'win32'") + .unwrap(); + let expected = Requirement::from_str( + "pytest; ('4.0' >= python_version or sys_platform == 'win32') and extra == 'dotenv'", + ) + .unwrap(); + let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?); + assert_eq!(actual, expected); + + Ok(()) + } } diff --git a/crates/pypi-types/Cargo.toml b/crates/pypi-types/Cargo.toml index 31a6dc13e1f49..3df43e7fc0c9b 100644 --- a/crates/pypi-types/Cargo.toml +++ b/crates/pypi-types/Cargo.toml @@ -18,12 +18,14 @@ pep508_rs = { workspace = true, features = ["rkyv", "serde"] } uv-normalize = { workspace = true } chrono = { workspace = true, features = ["serde"] } +indexmap = { workspace = true, features = ["serde"] } mailparse = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } rkyv = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/pypi-types/src/metadata.rs b/crates/pypi-types/src/metadata.rs index 93e508ed1207c..b4d2d4c69fdbf 100644 --- a/crates/pypi-types/src/metadata.rs +++ b/crates/pypi-types/src/metadata.rs @@ -1,5 +1,6 @@ //! Derived from `pypi_types_crate`. +use indexmap::IndexMap; use std::io; use std::str::FromStr; @@ -26,7 +27,6 @@ use crate::LenientVersionSpecifiers; #[serde(rename_all = "kebab-case")] pub struct Metadata23 { // Mandatory fields - pub metadata_version: String, pub name: PackageName, pub version: Version, // Optional fields @@ -46,6 +46,9 @@ pub enum Error { /// mail parse error #[error(transparent)] MailParse(#[from] MailParseError), + /// TOML parse error + #[error(transparent)] + Toml(#[from] toml::de::Error), /// Metadata field not found #[error("metadata field {0} not found")] FieldNotFound(&'static str), @@ -86,9 +89,6 @@ impl Metadata23 { pub fn parse_metadata(content: &[u8]) -> Result { let headers = Headers::parse(content)?; - let metadata_version = headers - .get_first_value("Metadata-Version") - .ok_or(Error::FieldNotFound("Metadata-Version"))?; let name = PackageName::new( headers .get_first_value("Name") @@ -124,7 +124,6 @@ impl Metadata23 { .collect::>(); Ok(Self { - metadata_version, name, version, requires_dist, @@ -200,7 +199,62 @@ impl Metadata23 { .collect::>(); Ok(Self { - metadata_version, + name, + version, + requires_dist, + requires_python, + provides_extras, + }) + } + + /// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621. + pub fn parse_pyproject_toml(contents: &str) -> Result { + let pyproject_toml: PyProjectToml = toml::from_str(contents)?; + + let project = pyproject_toml + .project + .ok_or(Error::FieldNotFound("project"))?; + + // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file. + let dynamic = project.dynamic.unwrap_or_default(); + for field in dynamic { + match field.as_str() { + "dependencies" => return Err(Error::DynamicField("dependencies")), + "optional-dependencies" => { + return Err(Error::DynamicField("optional-dependencies")) + } + "requires-python" => return Err(Error::DynamicField("requires-python")), + "version" => return Err(Error::DynamicField("version")), + _ => (), + } + } + + let name = project.name; + let version = project.version.ok_or(Error::FieldNotFound("version"))?; + let requires_python = project.requires_python.map(VersionSpecifiers::from); + + // Extract the requirements. + let mut requires_dist = project + .dependencies + .unwrap_or_default() + .into_iter() + .map(Requirement::from) + .collect::>(); + + // Extract the optional dependencies. + let mut provides_extras: Vec = Vec::new(); + for (extra, requirements) in project.optional_dependencies.unwrap_or_default() { + requires_dist.extend( + requirements + .into_iter() + .map(Requirement::from) + .map(|requirement| requirement.with_extra_marker(&extra)) + .collect::>(), + ); + provides_extras.push(extra); + } + + Ok(Self { name, version, requires_dist, @@ -210,12 +264,42 @@ impl Metadata23 { } } +/// A `pyproject.toml` as specified in PEP 517. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct PyProjectToml { + /// Project metadata + pub(crate) project: Option, +} + +/// PEP 621 project metadata. +/// +/// This is a subset of the full metadata specification, and only includes the +/// fields that are relevant to dependency resolution. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Project { + /// The name of the project + pub(crate) name: PackageName, + /// The version of the project as supported by PEP 440 + pub(crate) version: Option, + /// The Python version requirements of the project + pub(crate) requires_python: Option, + /// Project dependencies + pub(crate) dependencies: Option>, + /// Optional dependencies + pub(crate) optional_dependencies: Option>>, + /// Specifies which fields listed by PEP 621 were intentionally unspecified + /// so another tool can/will provide such metadata dynamically. + pub(crate) dynamic: Option>, +} + /// Python Package Metadata 1.0 and later as specified in /// . /// /// This is a subset of the full metadata specification, and only includes the /// fields that have been consistent across all versions of the specification. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +#[derive(Deserialize, Debug, Clone, Eq, PartialEq)] #[serde(rename_all = "kebab-case")] pub struct Metadata10 { pub name: PackageName, @@ -303,19 +387,16 @@ mod tests { let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0"; let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "1.0"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0\nAuthor: 中文\n\n一个 Python 包"; let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "1.0"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); let s = "Metadata-Version: 1.0\nName: =?utf-8?q?foobar?=\nVersion: 1.0"; let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "1.0"); assert_eq!(meta.name, PackageName::from_str("foobar").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); @@ -340,7 +421,6 @@ mod tests { let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0"; let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "2.3"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); @@ -350,9 +430,88 @@ mod tests { let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo"; let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "2.3"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); } + + #[test] + fn test_parse_pyproject_toml() { + let s = r#" + [project] + name = "asdf" + "#; + let meta = Metadata23::parse_pyproject_toml(s); + assert!(matches!(meta, Err(Error::FieldNotFound("version")))); + + let s = r#" + [project] + name = "asdf" + dynamic = ["version"] + "#; + let meta = Metadata23::parse_pyproject_toml(s); + assert!(matches!(meta, Err(Error::DynamicField("version")))); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert!(meta.requires_python.is_none()); + assert!(meta.requires_dist.is_empty()); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert!(meta.requires_dist.is_empty()); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + dependencies = ["foo"] + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + dependencies = ["foo"] + + [project.optional-dependencies] + dotenv = ["bar"] + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert_eq!( + meta.requires_dist, + vec![ + "foo".parse().unwrap(), + "bar; extra == \"dotenv\"".parse().unwrap() + ] + ); + assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]); + } } diff --git a/crates/uv-distribution/src/error.rs b/crates/uv-distribution/src/error.rs index d53a7bfcd908b..95b313b686f95 100644 --- a/crates/uv-distribution/src/error.rs +++ b/crates/uv-distribution/src/error.rs @@ -61,8 +61,12 @@ pub enum Error { NotFound(PathBuf), #[error("The source distribution is missing a `PKG-INFO` file")] MissingPkgInfo, - #[error("The source distribution does not support static metadata")] + #[error("The source distribution does not support static metadata in `PKG-INFO`")] DynamicPkgInfo(#[source] pypi_types::Error), + #[error("The source distribution is missing a `pyproject.toml` file")] + MissingPyprojectToml, + #[error("The source distribution does not support static metadata in `pyproject.toml`")] + DynamicPyprojectToml(#[source] pypi_types::Error), #[error("Unsupported scheme in URL: {0}")] UnsupportedScheme(String), diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 6507d4272e7bc..a4b12d0dfc9f6 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -956,10 +956,10 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { ) -> Result, Error> { debug!("Preparing metadata for: {source}"); - // Attempt to read static metadata from the source distribution. + // Attempt to read static metadata from the `PKG-INFO` file. match read_pkg_info(source_root).await { Ok(metadata) => { - debug!("Found static metadata for: {source}"); + debug!("Found static `PKG-INFO` for: {source}"); // Validate the metadata. if let Some(name) = source.name() { @@ -974,7 +974,30 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { return Ok(Some(metadata)); } Err(err @ (Error::MissingPkgInfo | Error::DynamicPkgInfo(_))) => { - debug!("No static metadata available for: {source} ({err:?})"); + debug!("No static `PKG-INFO` available for: {source} ({err:?})"); + } + Err(err) => return Err(err), + } + + // Attempt to read static metadata from the `pyproject.toml`. + match read_pyproject_toml(source_root).await { + Ok(metadata) => { + debug!("Found static `pyproject.toml` for: {source}"); + + // Validate the metadata. + if let Some(name) = source.name() { + if metadata.name != *name { + return Err(Error::NameMismatch { + metadata: metadata.name, + given: name.clone(), + }); + } + } + + return Ok(Some(metadata)); + } + Err(err @ (Error::MissingPyprojectToml | Error::DynamicPyprojectToml(_))) => { + debug!("No static `pyproject.toml` available for: {source} ({err:?})"); } Err(err) => return Err(err), } @@ -1106,6 +1129,25 @@ pub(crate) async fn read_pkg_info(source_tree: &Path) -> Result Result { + // Read the `pyproject.toml` file. + let content = match fs::read_to_string(source_tree.join("pyproject.toml")).await { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::MissingPyprojectToml); + } + Err(err) => return Err(Error::CacheRead(err)), + }; + + // Parse the metadata. + let metadata = + Metadata23::parse_pyproject_toml(&content).map_err(Error::DynamicPyprojectToml)?; + + Ok(metadata) +} + /// Read an existing HTTP-cached [`Manifest`], if it exists. pub(crate) fn read_http_manifest(cache_entry: &CacheEntry) -> Result, Error> { match fs_err::File::open(cache_entry.path()) {