Skip to content

Commit

Permalink
Add explicit index support
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Sep 18, 2024
1 parent e5dd67f commit 35df820
Show file tree
Hide file tree
Showing 18 changed files with 363 additions and 50 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/distribution-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ pub use crate::hash::*;
pub use crate::id::*;
pub use crate::index_url::*;
pub use crate::installed::*;
pub use crate::named_index::*;
pub use crate::prioritized_distribution::*;
pub use crate::resolution::*;
pub use crate::resolved::*;
Expand All @@ -75,6 +76,7 @@ mod hash;
mod id;
mod index_url;
mod installed;
mod named_index;
mod prioritized_distribution;
mod resolution;
mod resolved;
Expand Down
22 changes: 22 additions & 0 deletions crates/distribution-types/src/named_index.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
use url::Url;

#[derive(Debug, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct IndexSource {
pub name: String,
pub index: Url,
#[serde(default)]
pub kind: IndexKind,
}

#[derive(
Default, Debug, Copy, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize,
)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub enum IndexKind {
/// A PEP 503 and/or PEP 691-compliant index.
#[default]
Simple,
/// An index containing a list of links to distributions (e.g., `--find-links`).
Flat,
}
4 changes: 2 additions & 2 deletions crates/pypi-types/src/requirement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ pub enum RequirementSource {
Registry {
specifier: VersionSpecifiers,
/// Choose a version from the index with this name.
index: Option<String>,
index: Option<Url>,
},
// TODO(konsti): Track and verify version specifier from `project.dependencies` matches the
// version in remote location.
Expand Down Expand Up @@ -607,7 +607,7 @@ enum RequirementSourceWire {
Registry {
#[serde(skip_serializing_if = "VersionSpecifiers::is_empty", default)]
specifier: VersionSpecifiers,
index: Option<String>,
index: Option<Url>,
},
}

Expand Down
10 changes: 9 additions & 1 deletion crates/uv-client/src/registry_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use std::str::FromStr;
use async_http_range_reader::AsyncHttpRangeReader;
use futures::{FutureExt, TryStreamExt};
use http::HeaderMap;
use itertools::Either;
use reqwest::{Client, Response, StatusCode};
use reqwest_middleware::ClientWithMiddleware;
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -205,8 +206,15 @@ impl RegistryClient {
pub async fn simple(
&self,
package_name: &PackageName,
index: Option<&IndexUrl>,
) -> Result<Vec<(IndexUrl, OwnedArchive<SimpleMetadata>)>, Error> {
let mut it = self.index_urls.indexes().peekable();
let indexes = if let Some(index) = index {
Either::Left(std::iter::once(index))
} else {
Either::Right(self.index_urls.indexes())
};

let mut it = indexes.peekable();
if it.peek().is_none() {
return Err(ErrorKind::NoIndex(package_name.to_string()).into());
}
Expand Down
37 changes: 34 additions & 3 deletions crates/uv-distribution/src/metadata/lowering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use thiserror::Error;
use url::Url;

use distribution_filename::DistExtension;
use distribution_types::IndexSource;
use pep440_rs::VersionSpecifiers;
use pep508_rs::{VerbatimUrl, VersionOrUrl};
use pypi_types::{ParsedUrlError, Requirement, RequirementSource, VerbatimParsedUrl};
Expand Down Expand Up @@ -33,6 +34,7 @@ impl LoweredRequirement {
project_name: &PackageName,
project_dir: &Path,
project_sources: &BTreeMap<PackageName, Source>,
project_indexes: &[IndexSource],
workspace: &Workspace,
) -> Result<Self, LoweringError> {
let (source, origin) = if let Some(source) = project_sources.get(&requirement.name) {
Expand Down Expand Up @@ -108,7 +110,24 @@ impl LoweredRequirement {
editable.unwrap_or(false),
)?
}
Source::Registry { index } => registry_source(&requirement, index)?,
Source::Registry { index } => {
// Identify the named index from either the project indexes or the workspace indexes,
// in that order.
let Some(index) = project_indexes
.iter()
.find(|IndexSource { name, .. }| *name == index)
.or_else(|| {
workspace
.indexes()
.iter()
.find(|IndexSource { name, .. }| *name == index)
})
.map(|IndexSource { index, .. }| index.clone())
else {
return Err(LoweringError::MissingIndex(requirement.name, index));
};
registry_source(&requirement, index)?
}
Source::Workspace {
workspace: is_workspace,
} => {
Expand Down Expand Up @@ -185,6 +204,7 @@ impl LoweredRequirement {
requirement: pep508_rs::Requirement<VerbatimParsedUrl>,
dir: &Path,
sources: &BTreeMap<PackageName, Source>,
indexes: &[IndexSource],
) -> Result<Self, LoweringError> {
let source = sources.get(&requirement.name).cloned();

Expand Down Expand Up @@ -217,7 +237,16 @@ impl LoweredRequirement {
}
path_source(path, Origin::Project, dir, dir, editable.unwrap_or(false))?
}
Source::Registry { index } => registry_source(&requirement, index)?,
Source::Registry { index } => {
let Some(index) = indexes
.iter()
.find(|IndexSource { name, .. }| *name == index)
.map(|IndexSource { index, .. }| index.clone())
else {
return Err(LoweringError::MissingIndex(requirement.name, index));
};
registry_source(&requirement, index)?
}
Source::Workspace { .. } => {
return Err(LoweringError::WorkspaceMember);
}
Expand Down Expand Up @@ -252,6 +281,8 @@ pub enum LoweringError {
MoreThanOneGitRef,
#[error("Unable to combine options in `tool.uv.sources`")]
InvalidEntry,
#[error("Package `{0}` references an undeclared index: `{1}`")]
MissingIndex(PackageName, String),
#[error("Workspace members are not allowed in non-workspace contexts")]
WorkspaceMember,
#[error(transparent)]
Expand Down Expand Up @@ -342,7 +373,7 @@ fn url_source(url: Url, subdirectory: Option<PathBuf>) -> Result<RequirementSour
/// Convert a registry source into a [`RequirementSource`].
fn registry_source(
requirement: &pep508_rs::Requirement<VerbatimParsedUrl>,
index: String,
index: Url,
) -> Result<RequirementSource, LoweringError> {
match &requirement.version_or_url {
None => {
Expand Down
16 changes: 16 additions & 0 deletions crates/uv-distribution/src/metadata/requires_dist.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,20 @@ impl RequiresDist {
project_workspace: &ProjectWorkspace,
sources: SourceStrategy,
) -> Result<Self, MetadataError> {
// Collect any `tool.uv.index` entries.
let empty = vec![];
let indexes = match sources {
SourceStrategy::Enabled => project_workspace
.current_project()
.pyproject_toml()
.tool
.as_ref()
.and_then(|tool| tool.uv.as_ref())
.and_then(|uv| uv.index.as_deref())
.unwrap_or(&empty),
SourceStrategy::Disabled => &empty,
};

// Collect any `tool.uv.sources` and `tool.uv.dev_dependencies` from `pyproject.toml`.
let empty = BTreeMap::default();
let sources = match sources {
Expand Down Expand Up @@ -89,6 +103,7 @@ impl RequiresDist {
&metadata.name,
project_workspace.project_root(),
sources,
indexes,
project_workspace.workspace(),
)
.map(LoweredRequirement::into_inner)
Expand All @@ -112,6 +127,7 @@ impl RequiresDist {
&metadata.name,
project_workspace.project_root(),
sources,
indexes,
project_workspace.workspace(),
)
.map(LoweredRequirement::into_inner)
Expand Down
3 changes: 3 additions & 0 deletions crates/uv-resolver/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ pub enum ResolveError {
fork_markers: MarkerTree,
},

#[error("Requirements contain conflicting indexes for package `{0}`: `{1}` vs. `{2}`")]
ConflictingIndexes(PackageName, String, String),

#[error("Package `{0}` attempted to resolve via URL: {1}. URL dependencies must be expressed as direct requirements or constraints. Consider adding `{0} @ {1}` to your dependencies or constraints file.")]
DisallowedUrl(PackageName, String),

Expand Down
53 changes: 53 additions & 0 deletions crates/uv-resolver/src/resolver/indexes.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
use crate::{DependencyMode, Manifest, ResolveError, ResolverMarkers};
use distribution_types::IndexUrl;
use pep508_rs::{PackageName, VerbatimUrl};
use pypi_types::RequirementSource;
use rustc_hash::FxHashMap;
use std::collections::hash_map::Entry;

/// A map of package names to their explicit index across all forks.
#[derive(Debug, Default, Clone)]
pub(crate) struct Indexes(FxHashMap<PackageName, IndexUrl>);

impl Indexes {
/// Determine the set of explicit, pinned indexes in the [`Manifest`].
pub(crate) fn from_manifest(
manifest: &Manifest,
markers: &ResolverMarkers,
dependencies: DependencyMode,
) -> Result<Self, ResolveError> {
let mut indexes = FxHashMap::<PackageName, IndexUrl>::default();

for requirement in manifest.requirements(markers, dependencies) {
let RequirementSource::Registry {
index: Some(index), ..
} = &requirement.source
else {
continue;
};
let index = IndexUrl::from(VerbatimUrl::from_url(index.clone()));
match indexes.entry(requirement.name.clone()) {
Entry::Occupied(entry) => {
let existing = entry.get();
if *existing != index {
return Err(ResolveError::ConflictingIndexes(
requirement.name.clone(),
existing.to_string(),
index.to_string(),
));
}
}
Entry::Vacant(entry) => {
entry.insert(index);
}
}
}

Ok(Self(indexes))
}

/// Return the explicit index for a given [`PackageName`].
pub(crate) fn get(&self, package_name: &PackageName) -> Option<&IndexUrl> {
self.0.get(package_name)
}
}
Loading

0 comments on commit 35df820

Please sign in to comment.