diff --git a/Cargo.lock b/Cargo.lock index c0f65ce1bc18..a567dbf00c12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2750,6 +2750,7 @@ name = "pypi-types" version = "0.0.1" dependencies = [ "chrono", + "indexmap", "mailparse", "once_cell", "pep440_rs", @@ -2758,6 +2759,7 @@ dependencies = [ "rkyv", "serde", "thiserror", + "toml", "tracing", "url", "uv-normalize", diff --git a/crates/pep508-rs/src/lib.rs b/crates/pep508-rs/src/lib.rs index 1aa88cd34056..d37eff711dc9 100644 --- a/crates/pep508-rs/src/lib.rs +++ b/crates/pep508-rs/src/lib.rs @@ -471,6 +471,32 @@ impl Requirement { (true, Vec::new()) } } + + /// Return the requirement with an additional marker added, to require the given extra. + /// + /// For example, given `flask >= 2.0.2`, calling `with_extra_marker("dotenv")` would return + /// `flask >= 2.0.2 ; extra == "dotenv"`. + pub fn with_extra_marker(self, extra: &ExtraName) -> Self { + let marker = match self.marker { + Some(expression) => MarkerTree::And(vec![ + expression, + MarkerTree::Expression(MarkerExpression { + l_value: MarkerValue::Extra, + operator: MarkerOperator::Equal, + r_value: MarkerValue::QuotedString(extra.to_string()), + }), + ]), + None => MarkerTree::Expression(MarkerExpression { + l_value: MarkerValue::Extra, + operator: MarkerOperator::Equal, + r_value: MarkerValue::QuotedString(extra.to_string()), + }), + }; + Self { + marker: Some(marker), + ..self + } + } } impl UnnamedRequirement { @@ -1560,7 +1586,7 @@ mod tests { use insta::assert_snapshot; use pep440_rs::{Operator, Version, VersionPattern, VersionSpecifier}; - use uv_normalize::{ExtraName, PackageName}; + use uv_normalize::{ExtraName, InvalidNameError, PackageName}; use crate::marker::{ parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, @@ -2264,4 +2290,30 @@ mod tests { Ok(()) } + + #[test] + fn add_extra_marker() -> Result<(), InvalidNameError> { + let requirement = Requirement::from_str("pytest").unwrap(); + let expected = Requirement::from_str("pytest; extra == 'dotenv'").unwrap(); + let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?); + assert_eq!(actual, expected); + + let requirement = Requirement::from_str("pytest; '4.0' >= python_version").unwrap(); + let expected = + Requirement::from_str("pytest; '4.0' >= python_version and extra == 'dotenv'").unwrap(); + let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?); + assert_eq!(actual, expected); + + let requirement = + Requirement::from_str("pytest; '4.0' >= python_version or sys_platform == 'win32'") + .unwrap(); + let expected = Requirement::from_str( + "pytest; ('4.0' >= python_version or sys_platform == 'win32') and extra == 'dotenv'", + ) + .unwrap(); + let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?); + assert_eq!(actual, expected); + + Ok(()) + } } diff --git a/crates/pypi-types/Cargo.toml b/crates/pypi-types/Cargo.toml index 31a6dc13e1f4..3df43e7fc0c9 100644 --- a/crates/pypi-types/Cargo.toml +++ b/crates/pypi-types/Cargo.toml @@ -18,12 +18,14 @@ pep508_rs = { workspace = true, features = ["rkyv", "serde"] } uv-normalize = { workspace = true } chrono = { workspace = true, features = ["serde"] } +indexmap = { workspace = true, features = ["serde"] } mailparse = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } rkyv = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/pypi-types/src/metadata.rs b/crates/pypi-types/src/metadata.rs index 93e508ed1207..f3b267a00099 100644 --- a/crates/pypi-types/src/metadata.rs +++ b/crates/pypi-types/src/metadata.rs @@ -1,5 +1,6 @@ //! Derived from `pypi_types_crate`. +use indexmap::IndexMap; use std::io; use std::str::FromStr; @@ -22,11 +23,10 @@ use crate::LenientVersionSpecifiers; /// fields that are relevant to dependency resolution. /// /// At present, we support up to version 2.3 of the metadata specification. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "kebab-case")] pub struct Metadata23 { // Mandatory fields - pub metadata_version: String, pub name: PackageName, pub version: Version, // Optional fields @@ -46,6 +46,9 @@ pub enum Error { /// mail parse error #[error(transparent)] MailParse(#[from] MailParseError), + /// TOML parse error + #[error(transparent)] + Toml(#[from] toml::de::Error), /// Metadata field not found #[error("metadata field {0} not found")] FieldNotFound(&'static str), @@ -86,9 +89,6 @@ impl Metadata23 { pub fn parse_metadata(content: &[u8]) -> Result { let headers = Headers::parse(content)?; - let metadata_version = headers - .get_first_value("Metadata-Version") - .ok_or(Error::FieldNotFound("Metadata-Version"))?; let name = PackageName::new( headers .get_first_value("Name") @@ -124,7 +124,6 @@ impl Metadata23 { .collect::>(); Ok(Self { - metadata_version, name, version, requires_dist, @@ -200,7 +199,6 @@ impl Metadata23 { .collect::>(); Ok(Self { - metadata_version, name, version, requires_dist, @@ -208,6 +206,94 @@ impl Metadata23 { provides_extras, }) } + + /// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621. + pub fn parse_pyproject_toml(contents: &str) -> Result { + let pyproject_toml: PyProjectToml = toml::from_str(contents)?; + + let project = pyproject_toml + .project + .ok_or(Error::FieldNotFound("project"))?; + + // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file. + let dynamic = project.dynamic.unwrap_or_default(); + for field in dynamic { + match field.as_str() { + "dependencies" => return Err(Error::DynamicField("dependencies")), + "optional-dependencies" => { + return Err(Error::DynamicField("optional-dependencies")) + } + "requires-python" => return Err(Error::DynamicField("requires-python")), + "version" => return Err(Error::DynamicField("version")), + _ => (), + } + } + + let name = project.name; + let version = project.version.ok_or(Error::FieldNotFound("version"))?; + let requires_python = project.requires_python.map(VersionSpecifiers::from); + + // Extract the requirements. + let mut requires_dist = project + .dependencies + .unwrap_or_default() + .into_iter() + .map(Requirement::from) + .collect::>(); + + // Extract the optional dependencies. + let mut provides_extras: Vec = Vec::new(); + for (extra, requirements) in project.optional_dependencies.unwrap_or_default() { + requires_dist.extend( + requirements + .into_iter() + .map(Requirement::from) + .map(|requirement| requirement.with_extra_marker(&extra)) + .collect::>(), + ); + provides_extras.push(extra); + } + + Ok(Self { + name, + version, + requires_dist, + requires_python, + provides_extras, + }) + } +} + +/// A `pyproject.toml` as specified in PEP 517. +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct PyProjectToml { + /// Project metadata + pub(crate) project: Option, +} + +/// PEP 621 project metadata. +/// +/// This is a subset of the full metadata specification, and only includes the fields that are +/// relevant for dependency resolution. +/// +/// See . +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Project { + /// The name of the project + pub(crate) name: PackageName, + /// The version of the project as supported by PEP 440 + pub(crate) version: Option, + /// The Python version requirements of the project + pub(crate) requires_python: Option, + /// Project dependencies + pub(crate) dependencies: Option>, + /// Optional dependencies + pub(crate) optional_dependencies: Option>>, + /// Specifies which fields listed by PEP 621 were intentionally unspecified + /// so another tool can/will provide such metadata dynamically. + pub(crate) dynamic: Option>, } /// Python Package Metadata 1.0 and later as specified in @@ -215,7 +301,7 @@ impl Metadata23 { /// /// This is a subset of the full metadata specification, and only includes the /// fields that have been consistent across all versions of the specification. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +#[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "kebab-case")] pub struct Metadata10 { pub name: PackageName, @@ -303,19 +389,16 @@ mod tests { let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0"; let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "1.0"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0\nAuthor: 中文\n\n一个 Python 包"; let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "1.0"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); let s = "Metadata-Version: 1.0\nName: =?utf-8?q?foobar?=\nVersion: 1.0"; let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "1.0"); assert_eq!(meta.name, PackageName::from_str("foobar").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); @@ -340,7 +423,6 @@ mod tests { let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0"; let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "2.3"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); @@ -350,9 +432,88 @@ mod tests { let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo"; let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); - assert_eq!(meta.metadata_version, "2.3"); assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); assert_eq!(meta.version, Version::new([1, 0])); assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); } + + #[test] + fn test_parse_pyproject_toml() { + let s = r#" + [project] + name = "asdf" + "#; + let meta = Metadata23::parse_pyproject_toml(s); + assert!(matches!(meta, Err(Error::FieldNotFound("version")))); + + let s = r#" + [project] + name = "asdf" + dynamic = ["version"] + "#; + let meta = Metadata23::parse_pyproject_toml(s); + assert!(matches!(meta, Err(Error::DynamicField("version")))); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert!(meta.requires_python.is_none()); + assert!(meta.requires_dist.is_empty()); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert!(meta.requires_dist.is_empty()); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + dependencies = ["foo"] + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + dependencies = ["foo"] + + [project.optional-dependencies] + dotenv = ["bar"] + "#; + let meta = Metadata23::parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert_eq!( + meta.requires_dist, + vec![ + "foo".parse().unwrap(), + "bar; extra == \"dotenv\"".parse().unwrap() + ] + ); + assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]); + } } diff --git a/crates/uv-build/src/lib.rs b/crates/uv-build/src/lib.rs index 4c287b447301..0606c371f4ab 100644 --- a/crates/uv-build/src/lib.rs +++ b/crates/uv-build/src/lib.rs @@ -18,7 +18,7 @@ use once_cell::sync::Lazy; use regex::Regex; use rustc_hash::FxHashMap; use serde::de::{value, SeqAccess, Visitor}; -use serde::{de, Deserialize, Deserializer, Serialize}; +use serde::{de, Deserialize, Deserializer}; use tempfile::{tempdir_in, TempDir}; use thiserror::Error; use tokio::process::Command; @@ -26,7 +26,7 @@ use tokio::sync::Mutex; use tracing::{debug, info_span, instrument, Instrument}; use distribution_types::Resolution; -use pep440_rs::{Version, VersionSpecifiers}; +use pep440_rs::Version; use pep508_rs::{PackageName, Requirement}; use uv_fs::{PythonExt, Simplified}; use uv_interpreter::{Interpreter, PythonEnvironment}; @@ -193,8 +193,8 @@ impl Error { } } -/// A pyproject.toml as specified in PEP 517. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +/// A `pyproject.toml` as specified in PEP 517. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub struct PyProjectToml { /// Build-related data @@ -204,22 +204,23 @@ pub struct PyProjectToml { } /// The `[project]` section of a pyproject.toml as specified in PEP 621. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +/// +/// This representation only includes a subset of the fields defined in PEP 621 necessary for +/// informing wheel builds. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub struct Project { /// The name of the project pub name: PackageName, /// The version of the project as supported by PEP 440 pub version: Option, - /// The Python version requirements of the project - pub requires_python: Option, /// Specifies which fields listed by PEP 621 were intentionally unspecified so another tool /// can/will provide such metadata dynamically. pub dynamic: Option>, } /// The `[build-system]` section of a pyproject.toml as specified in PEP 517. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub struct BuildSystem { /// PEP 508 dependencies required to execute the build system. @@ -237,8 +238,7 @@ impl BackendPath { } } -#[derive(Serialize, Debug, Clone, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct BackendPath(Vec); impl<'de> Deserialize<'de> for BackendPath { diff --git a/crates/uv-distribution/src/error.rs b/crates/uv-distribution/src/error.rs index d53a7bfcd908..95b313b686f9 100644 --- a/crates/uv-distribution/src/error.rs +++ b/crates/uv-distribution/src/error.rs @@ -61,8 +61,12 @@ pub enum Error { NotFound(PathBuf), #[error("The source distribution is missing a `PKG-INFO` file")] MissingPkgInfo, - #[error("The source distribution does not support static metadata")] + #[error("The source distribution does not support static metadata in `PKG-INFO`")] DynamicPkgInfo(#[source] pypi_types::Error), + #[error("The source distribution is missing a `pyproject.toml` file")] + MissingPyprojectToml, + #[error("The source distribution does not support static metadata in `pyproject.toml`")] + DynamicPyprojectToml(#[source] pypi_types::Error), #[error("Unsupported scheme in URL: {0}")] UnsupportedScheme(String), diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index b5eae480ebd1..1ba819a4b190 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -955,10 +955,10 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { ) -> Result, Error> { debug!("Preparing metadata for: {source}"); - // Attempt to read static metadata from the source distribution. + // Attempt to read static metadata from the `PKG-INFO` file. match read_pkg_info(source_root).await { Ok(metadata) => { - debug!("Found static metadata for: {source}"); + debug!("Found static `PKG-INFO` for: {source}"); // Validate the metadata. if let Some(name) = source.name() { @@ -973,7 +973,30 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { return Ok(Some(metadata)); } Err(err @ (Error::MissingPkgInfo | Error::DynamicPkgInfo(_))) => { - debug!("No static metadata available for: {source} ({err:?})"); + debug!("No static `PKG-INFO` available for: {source} ({err:?})"); + } + Err(err) => return Err(err), + } + + // Attempt to read static metadata from the `pyproject.toml`. + match read_pyproject_toml(source_root).await { + Ok(metadata) => { + debug!("Found static `pyproject.toml` for: {source}"); + + // Validate the metadata. + if let Some(name) = source.name() { + if metadata.name != *name { + return Err(Error::NameMismatch { + metadata: metadata.name, + given: name.clone(), + }); + } + } + + return Ok(Some(metadata)); + } + Err(err @ (Error::MissingPyprojectToml | Error::DynamicPyprojectToml(_))) => { + debug!("No static `pyproject.toml` available for: {source} ({err:?})"); } Err(err) => return Err(err), } @@ -1105,6 +1128,25 @@ pub(crate) async fn read_pkg_info(source_tree: &Path) -> Result Result { + // Read the `pyproject.toml` file. + let content = match fs::read_to_string(source_tree.join("pyproject.toml")).await { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::MissingPyprojectToml); + } + Err(err) => return Err(Error::CacheRead(err)), + }; + + // Parse the metadata. + let metadata = + Metadata23::parse_pyproject_toml(&content).map_err(Error::DynamicPyprojectToml)?; + + Ok(metadata) +} + /// Read an existing HTTP-cached [`Manifest`], if it exists. pub(crate) fn read_http_manifest(cache_entry: &CacheEntry) -> Result, Error> { match fs_err::File::open(cache_entry.path()) { diff --git a/crates/uv-requirements/src/pyproject.rs b/crates/uv-requirements/src/pyproject.rs index ad2273805932..ea0a499a6f75 100644 --- a/crates/uv-requirements/src/pyproject.rs +++ b/crates/uv-requirements/src/pyproject.rs @@ -4,11 +4,12 @@ use serde::{Deserialize, Serialize}; use std::str::FromStr; use pep508_rs::Requirement; +use pypi_types::LenientRequirement; use uv_normalize::{ExtraName, PackageName}; use crate::ExtrasSpecification; -/// A pyproject.toml as specified in PEP 517 +/// A `pyproject.toml` as specified in PEP 517. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub(crate) struct PyProjectToml { @@ -16,7 +17,12 @@ pub(crate) struct PyProjectToml { pub(crate) project: Option, } -/// PEP 621 project metadata +/// PEP 621 project metadata. +/// +/// This is a subset of the full metadata specification, and only includes the fields that are +/// relevant for extracting static requirements. +/// +/// See . #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub(crate) struct Project { @@ -80,7 +86,7 @@ impl Pep621Metadata { .unwrap_or_default() .iter() .map(String::as_str) - .map(Requirement::from_str) + .map(|s| LenientRequirement::from_str(s).map(Requirement::from)) .collect::, _>>()?; // Include any optional dependencies specified in `extras`. @@ -94,7 +100,7 @@ impl Pep621Metadata { let requirements = requirements .iter() .map(String::as_str) - .map(Requirement::from_str) + .map(|s| LenientRequirement::from_str(s).map(Requirement::from)) .collect::, _>>()?; Ok::<(ExtraName, Vec), Pep621Error>((extra, requirements)) }) diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs index eaeec719993e..8f8a0039a601 100644 --- a/crates/uv/tests/pip_install.rs +++ b/crates/uv/tests/pip_install.rs @@ -881,7 +881,6 @@ fn install_editable_no_binary() { fn reinstall_build_system() -> Result<()> { let context = TestContext::new("3.12"); - // Install devpi. let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {r" flit_core<4.0.0 @@ -900,7 +899,7 @@ fn reinstall_build_system() -> Result<()> { ----- stderr ----- Resolved 8 packages in [TIME] - Downloaded 7 packages in [TIME] + Downloaded 8 packages in [TIME] Installed 8 packages in [TIME] + blinker==1.7.0 + click==8.1.7