Skip to content

Commit

Permalink
Add implicit indexes
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Sep 18, 2024
1 parent d213730 commit 98c909f
Show file tree
Hide file tree
Showing 14 changed files with 578 additions and 99 deletions.
70 changes: 70 additions & 0 deletions crates/distribution-types/src/index_source.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
use crate::IndexUrl;

#[derive(Debug, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct IndexSource {
/// The name of the index.
///
/// Index names can be used to reference indexes elsewhere in the configuration. For example,
/// you can pin a package to a specific index by name:
///
/// ```toml
/// [[tool.uv.index]]
/// name = "pytorch"
/// index = "https://download.pytorch.org/whl/cu121"
///
/// [tool.uv.sources]
/// torch = { index = "pytorch" }
/// ```
pub name: Option<String>,
/// The URL of the index.
///
/// Expects to receive a URL (e.g., `https://pypi.org/simple`) or a local path.
pub url: IndexUrl,
/// Mark the index as explicit.
///
/// Explicit indexes will _only_ be used when explicitly enabled via a `[tool.uv.sources]`
/// definition, as in:
///
/// ```toml
/// [[tool.uv.index]]
/// name = "pytorch"
/// index = "https://download.pytorch.org/whl/cu121"
/// explicit = true
///
/// [tool.uv.sources]
/// torch = { index = "pytorch" }
/// ```
#[serde(default)]
pub explicit: bool,
/// Mark the index as the default index.
///
/// By default, uv uses PyPI as the default index, such that even if additional indexes are
/// defined via `[[tool.uv.index]]`, PyPI will still be used as a fallback for packages that
/// aren't found elsewhere. To disable the PyPI default, set `default = true` on at least one
/// other index.
///
/// Marking an index as default will move it to the front of the list of indexes, such that it
/// is given the highest priority when resolving packages.
#[serde(default)]
pub default: bool,
// /// The type of the index.
// ///
// /// Indexes can either be PEP 503-compliant (i.e., a registry implementing the Simple API) or
// /// structured as a flat list of distributions (e.g., `--find-links`). In both cases, indexes
// /// can point to either local or remote resources.
// #[serde(default)]
// pub r#type: IndexKind,
}

// #[derive(
// Default, Debug, Copy, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize,
// )]
// #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
// pub enum IndexKind {
// /// A PEP 503 and/or PEP 691-compliant index.
// #[default]
// Simple,
// /// An index containing a list of links to distributions (e.g., `--find-links`).
// Flat,
// }
84 changes: 44 additions & 40 deletions crates/distribution-types/src/index_url.rs
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ impl From<VerbatimUrl> for FlatIndexLocation {
/// The index locations to use for fetching packages. By default, uses the PyPI index.
///
/// From a pip perspective, this type merges `--index-url`, `--extra-index-url`, and `--find-links`.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[derive(Default, Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "kebab-case", deny_unknown_fields)]
pub struct IndexLocations {
sources: Vec<IndexSource>,
Expand All @@ -307,19 +307,6 @@ pub struct IndexLocations {
no_index: bool,
}

impl Default for IndexLocations {
/// By default, use the `PyPI` index.
fn default() -> Self {
Self {
sources: Vec::new(),
index: Some(DEFAULT_INDEX_URL.clone()),
extra_index: Vec::new(),
flat_index: Vec::new(),
no_index: false,
}
}
}

impl IndexLocations {
/// Determine the index URLs to use for fetching packages.
pub fn new(
Expand Down Expand Up @@ -365,15 +352,32 @@ impl IndexLocations {
/// Returns `true` if no index configuration is set, i.e., the [`IndexLocations`] matches the
/// default configuration.
pub fn is_none(&self) -> bool {
self.sources.is_empty()
&& self.index.is_none()
&& self.extra_index.is_empty()
&& self.flat_index.is_empty()
&& !self.no_index
*self == Self::default()
}
}

impl<'a> IndexLocations {
/// Return an iterator over the `tool.uv.index` sources, prioritizing the default index.
fn sources(&'a self) -> impl Iterator<Item = &'a IndexUrl> + 'a {
if self.no_index {
Either::Left(std::iter::empty())
} else {
Either::Right(
self.sources
.iter()
.filter(|source| !source.explicit)
.filter(|source| source.default)
.chain(
self.sources
.iter()
.filter(|source| !source.explicit)
.filter(|source| !source.default),
)
.map(|source| &source.url),
)
}
}

/// Return the primary [`IndexUrl`] entry.
///
/// If `--no-index` is set, return `None`.
Expand All @@ -385,6 +389,7 @@ impl<'a> IndexLocations {
} else {
match self.index.as_ref() {
Some(index) => Some(index),
None if self.sources.iter().any(|source| source.default) => None,
None => Some(&DEFAULT_INDEX_URL),
}
}
Expand All @@ -401,7 +406,7 @@ impl<'a> IndexLocations {

/// Return an iterator over all [`IndexUrl`] entries.
pub fn indexes(&'a self) -> impl Iterator<Item = &'a IndexUrl> + 'a {
self.index().into_iter().chain(self.extra_index())
self.sources().chain(self.extra_index()).chain(self.index())
}

/// Return an iterator over the [`FlatIndexLocation`] entries.
Expand All @@ -428,43 +433,40 @@ impl<'a> IndexLocations {
pub fn urls(&'a self) -> impl Iterator<Item = &'a Url> + 'a {
self.indexes()
.map(IndexUrl::url)
.chain(self.flat_index.iter().filter_map(|index| match index {
FlatIndexLocation::Path(_) => None,
FlatIndexLocation::Url(url) => Some(url.raw()),
}))
.chain(self.flat_index().map(FlatIndexLocation::url))
}
}

/// The index URLs to use for fetching packages.
///
/// From a pip perspective, this type merges `--index-url` and `--extra-index-url`.
#[derive(Debug, Clone)]
#[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct IndexUrls {
sources: Vec<IndexSource>,
index: Option<IndexUrl>,
extra_index: Vec<IndexUrl>,
no_index: bool,
}

impl Default for IndexUrls {
/// By default, use the `PyPI` index.
fn default() -> Self {
Self {
sources: Vec::new(),
index: Some(DEFAULT_INDEX_URL.clone()),
extra_index: Vec::new(),
no_index: false,
}
}
}

impl<'a> IndexUrls {
/// Return an iterator over the `tool.uv.index` sources.
/// Return an iterator over the `tool.uv.index` sources, prioritizing the default index.
fn sources(&'a self) -> impl Iterator<Item = &'a IndexUrl> + 'a {
if self.no_index {
Either::Left(std::iter::empty())
} else {
Either::Right(self.sources.iter().map(|source| &source.index))
Either::Right(
self.sources
.iter()
.filter(|source| !source.explicit)
.filter(|source| source.default)
.chain(
self.sources
.iter()
.filter(|source| !source.explicit)
.filter(|source| !source.default),
)
.map(|source| &source.url),
)
}
}

Expand All @@ -479,6 +481,7 @@ impl<'a> IndexUrls {
} else {
match self.index.as_ref() {
Some(index) => Some(index),
None if self.sources.iter().any(|source| source.default) => None,
None => Some(&DEFAULT_INDEX_URL),
}
}
Expand All @@ -495,7 +498,8 @@ impl<'a> IndexUrls {

/// Return an iterator over all [`IndexUrl`] entries in order.
///
/// Prioritizes the extra indexes over the main index.
/// Prioritizes the `[tool.uv.index]` definitions over the `--extra-index-url` definitions
/// over the `--index-url` definition.
///
/// If `no_index` was enabled, then this always returns an empty
/// iterator.
Expand Down
4 changes: 2 additions & 2 deletions crates/distribution-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ pub use crate::error::*;
pub use crate::file::*;
pub use crate::hash::*;
pub use crate::id::*;
pub use crate::index_source::*;
pub use crate::index_url::*;
pub use crate::installed::*;
pub use crate::named_index::*;
pub use crate::prioritized_distribution::*;
pub use crate::resolution::*;
pub use crate::resolved::*;
Expand All @@ -74,9 +74,9 @@ mod error;
mod file;
mod hash;
mod id;
mod index_source;
mod index_url;
mod installed;
mod named_index;
mod prioritized_distribution;
mod resolution;
mod resolved;
Expand Down
22 changes: 0 additions & 22 deletions crates/distribution-types/src/named_index.rs

This file was deleted.

19 changes: 11 additions & 8 deletions crates/uv-distribution/src/metadata/lowering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,15 @@ impl LoweredRequirement {
// in that order.
let Some(index) = project_indexes
.iter()
.find(|IndexSource { name, .. }| *name == index)
.find(|IndexSource { name, .. }| {
name.as_ref().is_some_and(|name| *name == index)
})
.or_else(|| {
workspace
.indexes()
.iter()
.find(|IndexSource { name, .. }| *name == index)
workspace.indexes().iter().find(|IndexSource { name, .. }| {
name.as_ref().is_some_and(|name| *name == index)
})
})
.map(|IndexSource { index, .. }| index.clone())
.map(|IndexSource { url: index, .. }| index.clone())
else {
return Err(LoweringError::MissingIndex(requirement.name, index));
};
Expand Down Expand Up @@ -240,8 +241,10 @@ impl LoweredRequirement {
Source::Registry { index } => {
let Some(index) = indexes
.iter()
.find(|IndexSource { name, .. }| *name == index)
.map(|IndexSource { index, .. }| index.clone())
.find(|IndexSource { name, .. }| {
name.as_ref().is_some_and(|name| *name == index)
})
.map(|IndexSource { url: index, .. }| index.clone())
else {
return Err(LoweringError::MissingIndex(requirement.name, index));
};
Expand Down
14 changes: 13 additions & 1 deletion crates/uv-resolver/src/resolver/indexes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,19 @@ use pypi_types::RequirementSource;
use rustc_hash::FxHashMap;
use std::collections::hash_map::Entry;

/// A map of package names to their explicit index across all forks.
/// A map of package names to their explicit index.
///
/// For example, given:
/// ```toml
/// [[tool.uv.index]]
/// name = "pytorch"
/// url = "https://download.pytorch.org/whl/cu121"
///
/// [tool.uv.sources]
/// torch = { index = "pytorch" }
/// ```
///
/// [`Indexes`] would contain a single entry mapping `torch` to `https://download.pytorch.org/whl/cu121`.
#[derive(Debug, Default, Clone)]
pub(crate) struct Indexes(FxHashMap<PackageName, IndexUrl>);

Expand Down
31 changes: 26 additions & 5 deletions crates/uv-settings/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -313,18 +313,39 @@ pub struct ResolverOptions {
#[serde(rename_all = "kebab-case")]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct ResolverInstallerOptions {
/// The URL of the Python package index (by default: <https://pypi.org/simple>).
/// The indexes to use when resolving dependencies.
///
/// Accepts either a repository compliant with [PEP 503](https://peps.python.org/pep-0503/)
/// (the simple repository API), or a local directory laid out in the same format.
///
/// The index provided by this setting is given lower priority than any indexes specified via
/// Indexes are considered in the order in which they're defined, such that the first-defined
/// index has the highest priority. Further, the indexes provided by this setting are given
/// higher priority than any indexes specified via [`index_url`](#index-url) or
/// [`extra_index_url`](#extra-index-url).
///
/// If an index is marked as `explicit = true`, it will be used exclusively for those
/// dependencies that select it explicitly via `[tool.uv.sources]`, as in:
///
/// ```toml
/// [[tool.uv.index]]
/// name = "pytorch"
/// index = "https://download.pytorch.org/whl/cu121"
/// explicit = true
///
/// [tool.uv.sources]
/// torch = { index = "pytorch" }
/// ```
///
/// If an index is marked as `default = true`, it will be moved to the front of the list of
/// the list of indexes, such that it is given the highest priority when resolving packages.
/// Additionally, marking an index as default will disable the PyPI default index.
#[option(
default = "\"https://pypi.org/simple\"",
value_type = "str",
default = "\"[]\"",
value_type = "dict",
example = r#"
index-url = "https://test.pypi.org/simple"
[[tool.uv.index]]
name = "pytorch"
url = "https://download.pytorch.org/whl/cu121"
"#
)]
pub index: Option<Vec<IndexSource>>,
Expand Down
Loading

0 comments on commit 98c909f

Please sign in to comment.