From 4cffabe899a08490d6ad1a29172fc2b4d43a0b4b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 26 Sep 2024 19:48:41 -0400 Subject: [PATCH] Compiling with unified source --- crates/distribution-types/src/index_source.rs | 11 +++ crates/distribution-types/src/index_url.rs | 80 ++++--------------- crates/uv-settings/src/settings.rs | 29 +++++++ crates/uv/src/commands/pip/compile.rs | 12 +-- crates/uv/src/commands/pip/install.rs | 10 ++- crates/uv/src/commands/pip/sync.rs | 10 ++- crates/uv/src/settings.rs | 54 ++++++++++--- 7 files changed, 115 insertions(+), 91 deletions(-) diff --git a/crates/distribution-types/src/index_source.rs b/crates/distribution-types/src/index_source.rs index e4eec40c785f..352c0ae6fca6 100644 --- a/crates/distribution-types/src/index_source.rs +++ b/crates/distribution-types/src/index_source.rs @@ -68,3 +68,14 @@ pub struct IndexSource { // /// An index containing a list of links to distributions (e.g., `--find-links`). // Flat, // } + +impl From for IndexSource { + fn from(url: IndexUrl) -> Self { + Self { + url, + name: None, + explicit: false, + default: false, + } + } +} diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs index 19fb35bff2f4..02968c5ddf60 100644 --- a/crates/distribution-types/src/index_url.rs +++ b/crates/distribution-types/src/index_url.rs @@ -300,9 +300,8 @@ impl From for FlatIndexLocation { #[derive(Default, Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub struct IndexLocations { - sources: Vec, - index: Option, - extra_index: Vec, + index: Option, + extra_index: Vec, flat_index: Vec, no_index: bool, } @@ -310,14 +309,12 @@ pub struct IndexLocations { impl IndexLocations { /// Determine the index URLs to use for fetching packages. pub fn new( - sources: Vec, - index: Option, - extra_index: Vec, + index: Option, + extra_index: Vec, flat_index: Vec, no_index: bool, ) -> Self { Self { - sources, index, extra_index, flat_index, @@ -334,14 +331,12 @@ impl IndexLocations { #[must_use] pub fn combine( self, - sources: Vec, - index: Option, - extra_index: Vec, + index: Option, + extra_index: Vec, flat_index: Vec, no_index: bool, ) -> Self { Self { - sources: self.sources.into_iter().chain(sources).collect(), index: self.index.or(index), extra_index: self.extra_index.into_iter().chain(extra_index).collect(), flat_index: self.flat_index.into_iter().chain(flat_index).collect(), @@ -357,27 +352,6 @@ impl IndexLocations { } impl<'a> IndexLocations { - /// Return an iterator over the `tool.uv.index` sources, prioritizing the default index. - fn sources(&'a self) -> impl Iterator + 'a { - if self.no_index { - Either::Left(std::iter::empty()) - } else { - Either::Right( - self.sources - .iter() - .filter(|source| !source.explicit) - .filter(|source| source.default) - .chain( - self.sources - .iter() - .filter(|source| !source.explicit) - .filter(|source| !source.default), - ) - .map(|source| &source.url), - ) - } - } - /// Return the primary [`IndexUrl`] entry. /// /// If `--no-index` is set, return `None`. @@ -388,8 +362,7 @@ impl<'a> IndexLocations { None } else { match self.index.as_ref() { - Some(index) => Some(index), - None if self.sources.iter().any(|source| source.default) => None, + Some(index) => Some(&index.url), None => Some(&DEFAULT_INDEX_URL), } } @@ -400,7 +373,7 @@ impl<'a> IndexLocations { if self.no_index { Either::Left(std::iter::empty()) } else { - Either::Right(self.extra_index.iter()) + Either::Right(self.extra_index.iter().map(|index| &index.url)) } } @@ -411,7 +384,7 @@ impl<'a> IndexLocations { /// If `no_index` was enabled, then this always returns an empty /// iterator. pub fn indexes(&'a self) -> impl Iterator + 'a { - self.sources().chain(self.extra_index()).chain(self.index()) + self.extra_index().chain(self.index()) } /// Return an iterator over the [`FlatIndexLocation`] entries. @@ -427,7 +400,6 @@ impl<'a> IndexLocations { /// Clone the index locations into a [`IndexUrls`] instance. pub fn index_urls(&'a self) -> IndexUrls { IndexUrls { - sources: self.sources.clone(), index: self.index.clone(), extra_index: self.extra_index.clone(), no_index: self.no_index, @@ -447,34 +419,12 @@ impl<'a> IndexLocations { /// From a pip perspective, this type merges `--index-url` and `--extra-index-url`. #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct IndexUrls { - sources: Vec, - index: Option, - extra_index: Vec, + index: Option, + extra_index: Vec, no_index: bool, } impl<'a> IndexUrls { - /// Return an iterator over the `tool.uv.index` sources, prioritizing the default index. - fn sources(&'a self) -> impl Iterator + 'a { - if self.no_index { - Either::Left(std::iter::empty()) - } else { - Either::Right( - self.sources - .iter() - .filter(|source| !source.explicit) - .filter(|source| source.default) - .chain( - self.sources - .iter() - .filter(|source| !source.explicit) - .filter(|source| !source.default), - ) - .map(|source| &source.url), - ) - } - } - /// Return the fallback [`IndexUrl`] entry. /// /// If `--no-index` is set, return `None`. @@ -485,8 +435,7 @@ impl<'a> IndexUrls { None } else { match self.index.as_ref() { - Some(index) => Some(index), - None if self.sources.iter().any(|source| source.default) => None, + Some(index) => Some(&index.url), None => Some(&DEFAULT_INDEX_URL), } } @@ -497,7 +446,7 @@ impl<'a> IndexUrls { if self.no_index { Either::Left(std::iter::empty()) } else { - Either::Right(self.extra_index.iter()) + Either::Right(self.extra_index.iter().map(|index| &index.url)) } } @@ -509,14 +458,13 @@ impl<'a> IndexUrls { /// If `no_index` was enabled, then this always returns an empty /// iterator. pub fn indexes(&'a self) -> impl Iterator + 'a { - self.sources().chain(self.extra_index()).chain(self.index()) + self.extra_index().chain(self.index()) } } impl From for IndexUrls { fn from(locations: IndexLocations) -> Self { Self { - sources: locations.sources, index: locations.index, extra_index: locations.extra_index, no_index: locations.no_index, diff --git a/crates/uv-settings/src/settings.rs b/crates/uv-settings/src/settings.rs index 55c3fac3bc83..aee371cd3bb3 100644 --- a/crates/uv-settings/src/settings.rs +++ b/crates/uv-settings/src/settings.rs @@ -742,6 +742,35 @@ pub struct PipOptions { "# )] pub prefix: Option, + /// The indexes to use when resolving dependencies. + /// + /// Accepts either a repository compliant with [PEP 503](https://peps.python.org/pep-0503/) + /// (the simple repository API), or a local directory laid out in the same format. + /// + /// Indexes are considered in the order in which they're defined, such that the first-defined + /// index has the highest priority. Further, the indexes provided by this setting are given + /// higher priority than any indexes specified via [`index_url`](#index-url) or + /// [`extra_index_url`](#extra-index-url). + /// + /// If an index is marked as `explicit = true`, it will be used exclusively for those + /// dependencies that select it explicitly via `[tool.uv.sources]`, as in: + /// + /// ```toml + /// [[tool.uv.index]] + /// name = "pytorch" + /// url = "https://download.pytorch.org/whl/cu121" + /// explicit = true + /// + /// [tool.uv.sources] + /// torch = { index = "pytorch" } + /// ``` + /// + /// If an index is marked as `default = true`, it will be moved to the front of the list of + /// the list of indexes, such that it is given the highest priority when resolving packages. + /// Additionally, marking an index as default will disable the PyPI default index. + #[serde(skip)] + #[cfg_attr(feature = "schemars", schemars(skip))] + pub index: Option>, /// The URL of the Python package index (by default: ). /// /// Accepts either a repository compliant with [PEP 503](https://peps.python.org/pep-0503/) diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index bbafac479156..092718c1b8c4 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -8,8 +8,8 @@ use owo_colors::OwoColorize; use tracing::debug; use distribution_types::{ - DependencyMetadata, IndexCapabilities, IndexLocations, NameRequirementSpecification, - UnresolvedRequirementSpecification, Verbatim, + DependencyMetadata, IndexCapabilities, IndexLocations, IndexSource, + NameRequirementSpecification, UnresolvedRequirementSpecification, Verbatim, }; use install_wheel_rs::linker::LinkMode; use pypi_types::{Requirement, SupportedEnvironments}; @@ -275,9 +275,11 @@ pub(crate) async fn pip_compile( // Incorporate any index locations from the provided sources. let index_locations = index_locations.combine( - Vec::default(), - index_url, - extra_index_urls, + index_url.map(IndexSource::from), + extra_index_urls + .into_iter() + .map(IndexSource::from) + .collect(), find_links, no_index, ); diff --git a/crates/uv/src/commands/pip/install.rs b/crates/uv/src/commands/pip/install.rs index d0e9dd9b727c..f1f3fb19572c 100644 --- a/crates/uv/src/commands/pip/install.rs +++ b/crates/uv/src/commands/pip/install.rs @@ -6,7 +6,7 @@ use owo_colors::OwoColorize; use tracing::{debug, enabled, Level}; use distribution_types::{ - DependencyMetadata, IndexLocations, NameRequirementSpecification, Resolution, + DependencyMetadata, IndexLocations, IndexSource, NameRequirementSpecification, Resolution, UnresolvedRequirementSpecification, }; use install_wheel_rs::linker::LinkMode; @@ -273,9 +273,11 @@ pub(crate) async fn pip_install( // Incorporate any index locations from the provided sources. let index_locations = index_locations.combine( - Vec::default(), - index_url, - extra_index_urls, + index_url.map(IndexSource::from), + extra_index_urls + .into_iter() + .map(IndexSource::from) + .collect(), find_links, no_index, ); diff --git a/crates/uv/src/commands/pip/sync.rs b/crates/uv/src/commands/pip/sync.rs index 098591f6b7d8..f6349691d617 100644 --- a/crates/uv/src/commands/pip/sync.rs +++ b/crates/uv/src/commands/pip/sync.rs @@ -5,7 +5,7 @@ use anyhow::Result; use owo_colors::OwoColorize; use tracing::debug; -use distribution_types::{DependencyMetadata, IndexLocations, Resolution}; +use distribution_types::{DependencyMetadata, IndexLocations, IndexSource, Resolution}; use install_wheel_rs::linker::LinkMode; use pep508_rs::PackageName; use uv_auth::store_credentials_from_url; @@ -216,9 +216,11 @@ pub(crate) async fn pip_sync( // Incorporate any index locations from the provided sources. let index_locations = index_locations.combine( - Vec::default(), - index_url, - extra_index_urls, + index_url.map(IndexSource::from), + extra_index_urls + .into_iter() + .map(IndexSource::from) + .collect(), find_links, no_index, ); diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index f085756b3cdc..2446017f1724 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::process; use std::str::FromStr; -use distribution_types::{DependencyMetadata, IndexLocations}; +use distribution_types::{DependencyMetadata, IndexLocations, IndexSource}; use install_wheel_rs::linker::LinkMode; use pep508_rs::{ExtraName, RequirementOrigin}; use pypi_types::{Requirement, SupportedEnvironments}; @@ -1899,9 +1899,19 @@ impl From for ResolverSettings { fn from(value: ResolverOptions) -> Self { Self { index_locations: IndexLocations::new( - value.index.unwrap_or_default(), - value.index_url, - value.extra_index_url.unwrap_or_default(), + value.index_url.map(IndexSource::from), + value + .index + .into_iter() + .flatten() + .chain( + value + .extra_index_url + .into_iter() + .flatten() + .map(IndexSource::from), + ) + .collect(), value.find_links.unwrap_or_default(), value.no_index.unwrap_or_default(), ), @@ -2027,9 +2037,19 @@ impl From for ResolverInstallerSettings { fn from(value: ResolverInstallerOptions) -> Self { Self { index_locations: IndexLocations::new( - value.index.unwrap_or_default(), - value.index_url, - value.extra_index_url.unwrap_or_default(), + value.index_url.map(IndexSource::from), + value + .index + .into_iter() + .flatten() + .chain( + value + .extra_index_url + .into_iter() + .flatten() + .map(IndexSource::from), + ) + .collect(), value.find_links.unwrap_or_default(), value.no_index.unwrap_or_default(), ), @@ -2135,6 +2155,7 @@ impl PipSettings { break_system_packages, target, prefix, + index, index_url, extra_index_url, no_index, @@ -2218,6 +2239,7 @@ impl PipSettings { // preferring the latter. // // For example, prefer `tool.uv.pip.index-url` over `tool.uv.index-url`. + let index = index.combine(top_level_index); let index_url = index_url.combine(top_level_index_url); let extra_index_url = extra_index_url.combine(top_level_extra_index_url); let no_index = no_index.combine(top_level_no_index); @@ -2243,11 +2265,19 @@ impl PipSettings { Self { index_locations: IndexLocations::new( - top_level_index.unwrap_or_default(), - args.index_url.combine(index_url), - args.extra_index_url - .combine(extra_index_url) - .unwrap_or_default(), + args.index_url.combine(index_url).map(IndexSource::from), + args.index + .combine(index) + .into_iter() + .flatten() + .chain( + args.extra_index_url + .combine(extra_index_url) + .into_iter() + .flatten() + .map(IndexSource::from), + ) + .collect(), args.find_links.combine(find_links).unwrap_or_default(), args.no_index.combine(no_index).unwrap_or_default(), ),