From 5db563138f3863bd65272bba1ab096bd96bc1deb Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Thu, 2 Nov 2023 00:23:04 -0400 Subject: [PATCH] Ensure canonicalization of extras when creating ireqs _compat now exports install_req_from_line and canonicalize_ireq --- piptools/_compat/__init__.py | 4 ++++ piptools/_compat/pip_compat.py | 23 +++++++++++++++++++++-- piptools/build.py | 8 ++++++-- piptools/resolver.py | 3 +-- piptools/scripts/compile.py | 3 +-- piptools/utils.py | 3 +-- 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index be1397092..ccc61eec3 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -3,8 +3,10 @@ from .pip_compat import ( PIP_VERSION, Distribution, + canonicalize_ireq, create_wheel_cache, get_dev_pkgs, + install_req_from_line, parse_requirements, ) @@ -12,6 +14,8 @@ "PIP_VERSION", "Distribution", "parse_requirements", + "install_req_from_line", "create_wheel_cache", "get_dev_pkgs", + "canonicalize_ireq", ] diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 6409fbc2a..6cf1eb2b3 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -2,7 +2,7 @@ import optparse from dataclasses import dataclass -from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast +from typing import TYPE_CHECKING, Any, Iterable, Iterator, Set, cast import pip from pip._internal.cache import WheelCache @@ -13,7 +13,11 @@ from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.req import parse_requirements as _parse_requirements +from pip._internal.req.constructors import ( + install_req_from_line as _install_req_from_line, +) from pip._internal.req.constructors import install_req_from_parsed_requirement +from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import Requirement @@ -63,6 +67,19 @@ def _from_importlib(cls, dist: _ImportLibDist) -> Distribution: return cls(dist._dist.name, dist._dist.version, requires, dist.direct_url) +def canonicalize_ireq(ireq: InstallRequirement) -> None: + if hasattr(ireq.req, "extras") and ireq.req.extras: + ireq.req.extras = set(map(canonicalize_name, ireq.req.extras)) + if hasattr(ireq, "extras") and ireq.extras: + ireq.extras = set(map(canonicalize_name, ireq.extras)) + + +def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: + ireq = _install_req_from_line(*args, **kwargs) + canonicalize_ireq(ireq) + return ireq + + def parse_requirements( filename: str, session: PipSession, @@ -74,7 +91,9 @@ def parse_requirements( for parsed_req in _parse_requirements( filename, session, finder=finder, options=options, constraint=constraint ): - yield install_req_from_parsed_requirement(parsed_req, isolated=isolated) + ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated) + canonicalize_ireq(ireq) + yield ireq def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache: diff --git a/piptools/build.py b/piptools/build.py index 6f87c32e1..73867f6a2 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -13,7 +13,9 @@ import build.env import pyproject_hooks from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line, parse_req_from_line +from pip._internal.req.constructors import parse_req_from_line + +from ._compat import canonicalize_ireq, install_req_from_line PYPROJECT_TOML = "pyproject.toml" @@ -139,13 +141,15 @@ def _prepare_requirements( replaced_package_name = req.replace(package_name, str(package_dir), 1) parts = parse_req_from_line(replaced_package_name, comes_from) - yield InstallRequirement( + ireq = InstallRequirement( parts.requirement, comes_from, link=parts.link, markers=parts.markers, extras=parts.extras, ) + canonicalize_ireq(ireq) + yield ireq def _prepare_build_requirements( diff --git a/piptools/resolver.py b/piptools/resolver.py index aaa8cd606..5c26ba333 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -14,7 +14,6 @@ update_env_context_manager, ) from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.resolvelib.base import Candidate from pip._internal.resolution.resolvelib.candidates import ExtrasCandidate from pip._internal.resolution.resolvelib.resolver import Resolver @@ -27,7 +26,7 @@ from piptools.cache import DependencyCache from piptools.repositories.base import BaseRepository -from ._compat import create_wheel_cache +from ._compat import create_wheel_cache, install_req_from_line from .exceptions import PipToolsError from .logging import log from .utils import ( diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 462215f4d..5838f0483 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -12,10 +12,9 @@ from build import BuildBackendException from click.utils import LazyFile, safecall from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url -from .._compat import parse_requirements +from .._compat import install_req_from_line, parse_requirements from ..build import build_project_metadata from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError diff --git a/piptools/utils.py b/piptools/utils.py index 62cb26a0b..d5ccd9c26 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -22,7 +22,6 @@ import click from click.utils import LazyFile from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.resolvelib.base import Requirement as PipRequirement from pip._internal.utils.misc import redact_auth_from_url from pip._internal.vcs import is_url @@ -33,7 +32,7 @@ from pip._vendor.packaging.version import Version from pip._vendor.pkg_resources import get_distribution -from piptools._compat import PIP_VERSION +from piptools._compat import PIP_VERSION, install_req_from_line from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from piptools.subprocess_utils import run_python_snippet