Skip to content

Build backend: Add direct builds to the resolver and installer #9621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Dec 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion crates/uv-bench/benches/uv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ mod resolver {
use uv_client::RegistryClient;
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, Constraints, IndexStrategy, LowerBound,
SourceStrategy,
PreviewMode, SourceStrategy,
};
use uv_dispatch::{BuildDispatch, SharedState};
use uv_distribution::DistributionDatabase;
Expand Down Expand Up @@ -190,6 +190,7 @@ mod resolver {
LowerBound::default(),
sources,
concurrency,
PreviewMode::Enabled,
);

let markers = if universal {
Expand Down
1 change: 1 addition & 0 deletions crates/uv-build-backend/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ uv-normalize = { workspace = true }
uv-pep440 = { workspace = true }
uv-pep508 = { workspace = true }
uv-pypi-types = { workspace = true }
uv-version = { workspace = true }
uv-warnings = { workspace = true }

csv = { workspace = true }
Expand Down
2 changes: 1 addition & 1 deletion crates/uv-build-backend/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ mod metadata;
mod source_dist;
mod wheel;

pub use metadata::PyProjectToml;
pub use metadata::{check_direct_build, PyProjectToml};
pub use source_dist::{build_source_dist, list_source_dist};
pub use wheel::{build_editable, build_wheel, list_wheel, metadata};

Expand Down
36 changes: 36 additions & 0 deletions crates/uv-build-backend/src/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use itertools::Itertools;
use serde::Deserialize;
use std::collections::{BTreeMap, Bound};
use std::ffi::OsStr;
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use tracing::{debug, trace};
Expand Down Expand Up @@ -50,6 +51,41 @@ pub enum ValidationError {
InvalidSpdx(String, #[source] spdx::error::ParseError),
}

/// Check if the build backend is matching the currently running uv version.
pub fn check_direct_build(source_tree: &Path, name: impl Display) -> bool {
let pyproject_toml: PyProjectToml =
match fs_err::read_to_string(source_tree.join("pyproject.toml"))
.map_err(|err| err.to_string())
.and_then(|pyproject_toml| {
toml::from_str(&pyproject_toml).map_err(|err| err.to_string())
}) {
Ok(pyproject_toml) => pyproject_toml,
Err(err) => {
debug!(
"Not using uv build backend direct build of {name}, no pyproject.toml: {err}"
);
return false;
}
};
match pyproject_toml
.check_build_system(uv_version::version())
.as_slice()
{
// No warnings -> match
[] => true,
// Any warning -> no match
[first, others @ ..] => {
debug!(
"Not using uv build backend direct build of {name}, pyproject.toml does not match: {first}"
);
for other in others {
trace!("Further uv build backend direct build of {name} mismatch: {other}");
}
false
}
}
}

/// A `pyproject.toml` as specified in PEP 517.
#[derive(Deserialize, Debug, Clone)]
#[serde(
Expand Down
6 changes: 3 additions & 3 deletions crates/uv-build-frontend/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ impl SourceBuild {
interpreter: &Interpreter,
build_context: &impl BuildContext,
source_build_context: SourceBuildContext,
version_id: Option<String>,
version_id: Option<&str>,
locations: &IndexLocations,
source_strategy: SourceStrategy,
config_settings: ConfigSettings,
Expand Down Expand Up @@ -376,7 +376,7 @@ impl SourceBuild {
build_context,
package_name.as_ref(),
package_version.as_ref(),
version_id.as_deref(),
version_id,
locations,
source_strategy,
build_kind,
Expand All @@ -401,7 +401,7 @@ impl SourceBuild {
metadata_directory: None,
package_name,
package_version,
version_id,
version_id: version_id.map(ToString::to_string),
environment_variables,
modified_path,
runner,
Expand Down
4 changes: 4 additions & 0 deletions crates/uv-dispatch/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ doctest = false
workspace = true

[dependencies]
uv-build-backend = { workspace = true }
uv-build-frontend = { workspace = true }
uv-cache = { workspace = true }
uv-client = { workspace = true }
uv-configuration = { workspace = true }
uv-distribution = { workspace = true }
uv-distribution-filename = { workspace = true }
uv-distribution-types = { workspace = true }
uv-git = { workspace = true }
uv-install-wheel = { workspace = true }
Expand All @@ -30,9 +32,11 @@ uv-pypi-types = { workspace = true }
uv-python = { workspace = true }
uv-resolver = { workspace = true }
uv-types = { workspace = true }
uv-version = { workspace = true }

anyhow = { workspace = true }
futures = { workspace = true }
itertools = { workspace = true }
rustc-hash = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
81 changes: 76 additions & 5 deletions crates/uv-dispatch/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,18 @@ use anyhow::{anyhow, Context, Result};
use futures::FutureExt;
use itertools::Itertools;
use rustc_hash::FxHashMap;
use tracing::{debug, instrument};

use tracing::{debug, instrument, trace};
use uv_build_backend::check_direct_build;
use uv_build_frontend::{SourceBuild, SourceBuildContext};
use uv_cache::Cache;
use uv_client::RegistryClient;
use uv_configuration::{
BuildKind, BuildOptions, ConfigSettings, Constraints, IndexStrategy, LowerBound, Reinstall,
SourceStrategy,
BuildKind, BuildOptions, ConfigSettings, Constraints, IndexStrategy, LowerBound, PreviewMode,
Reinstall, SourceStrategy,
};
use uv_configuration::{BuildOutput, Concurrency};
use uv_distribution::DistributionDatabase;
use uv_distribution_filename::DistFilename;
use uv_distribution_types::{
CachedDist, DependencyMetadata, IndexCapabilities, IndexLocations, Name, Resolution,
SourceDist, VersionOrUrlRef,
Expand Down Expand Up @@ -57,6 +58,7 @@ pub struct BuildDispatch<'a> {
bounds: LowerBound,
sources: SourceStrategy,
concurrency: Concurrency,
preview: PreviewMode,
}

impl<'a> BuildDispatch<'a> {
Expand All @@ -79,6 +81,7 @@ impl<'a> BuildDispatch<'a> {
bounds: LowerBound,
sources: SourceStrategy,
concurrency: Concurrency,
preview: PreviewMode,
) -> Self {
Self {
client,
Expand All @@ -101,6 +104,7 @@ impl<'a> BuildDispatch<'a> {
bounds,
sources,
concurrency,
preview,
}
}

Expand Down Expand Up @@ -340,7 +344,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
source: &'data Path,
subdirectory: Option<&'data Path>,
install_path: &'data Path,
version_id: Option<String>,
version_id: Option<&'data str>,
dist: Option<&'data SourceDist>,
sources: SourceStrategy,
build_kind: BuildKind,
Expand Down Expand Up @@ -394,6 +398,73 @@ impl<'a> BuildContext for BuildDispatch<'a> {
.await?;
Ok(builder)
}

async fn direct_build<'data>(
&'data self,
source: &'data Path,
subdirectory: Option<&'data Path>,
output_dir: &'data Path,
build_kind: BuildKind,
version_id: Option<&'data str>,
) -> Result<Option<DistFilename>> {
// Direct builds are a preview feature with the uv build backend.
if self.preview.is_disabled() {
trace!("Preview is disabled, not checking for direct build");
return Ok(None);
}

let source_tree = if let Some(subdir) = subdirectory {
source.join(subdir)
} else {
source.to_path_buf()
};

// Only perform the direct build if the backend is uv in a compatible version.
let source_tree_str = source_tree.display().to_string();
let identifier = version_id.unwrap_or_else(|| &source_tree_str);
if !check_direct_build(&source_tree, identifier) {
trace!("Requirements for direct build not matched: {identifier}");
return Ok(None);
}

debug!("Performing direct build for {identifier}");

let output_dir = output_dir.to_path_buf();
let filename = tokio::task::spawn_blocking(move || -> Result<_> {
let filename = match build_kind {
BuildKind::Wheel => {
let wheel = uv_build_backend::build_wheel(
&source_tree,
&output_dir,
None,
uv_version::version(),
)?;
DistFilename::WheelFilename(wheel)
}
BuildKind::Sdist => {
let source_dist = uv_build_backend::build_source_dist(
&source_tree,
&output_dir,
uv_version::version(),
)?;
DistFilename::SourceDistFilename(source_dist)
}
BuildKind::Editable => {
let wheel = uv_build_backend::build_editable(
&source_tree,
&output_dir,
None,
uv_version::version(),
)?;
DistFilename::WheelFilename(wheel)
}
};
Ok(filename)
})
.await??;

Ok(Some(filename))
}
}

/// Shared state used during resolution and installation.
Expand Down
42 changes: 31 additions & 11 deletions crates/uv-distribution/src/source/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1803,27 +1803,47 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
fs::create_dir_all(&cache_shard)
.await
.map_err(Error::CacheWrite)?;
let disk_filename = self
// Try a direct build if that isn't disabled and the uv build backend is used.
let disk_filename = if let Some(name) = self
.build_context
.setup_build(
.direct_build(
source_root,
subdirectory,
source_root,
Some(source.to_string()),
source.as_dist(),
source_strategy,
temp_dir.path(),
if source.is_editable() {
BuildKind::Editable
} else {
BuildKind::Wheel
},
BuildOutput::Debug,
Some(&source.to_string()),
)
.await
.map_err(Error::Build)?
.wheel(temp_dir.path())
.await
.map_err(Error::Build)?;
{
// In the uv build backend, the normalized filename and the disk filename are the same.
name.to_string()
} else {
self.build_context
.setup_build(
source_root,
subdirectory,
source_root,
Some(&source.to_string()),
source.as_dist(),
source_strategy,
if source.is_editable() {
BuildKind::Editable
} else {
BuildKind::Wheel
},
BuildOutput::Debug,
)
.await
.map_err(Error::Build)?
.wheel(temp_dir.path())
.await
.map_err(Error::Build)?
};

// Read the metadata from the wheel.
let filename = WheelFilename::from_str(&disk_filename)?;
Expand Down Expand Up @@ -1884,7 +1904,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
source_root,
subdirectory,
source_root,
Some(source.to_string()),
Some(&source.to_string()),
source.as_dist(),
source_strategy,
if source.is_editable() {
Expand Down
1 change: 1 addition & 0 deletions crates/uv-types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ workspace = true
[dependencies]
uv-cache = { workspace = true }
uv-configuration = { workspace = true }
uv-distribution-filename = { workspace = true }
uv-distribution-types = { workspace = true }
uv-git = { workspace = true }
uv-normalize = { workspace = true }
Expand Down
Loading
Loading