From fe6e838876dabaa95fcc0ffc5ec432cbf58d26d6 Mon Sep 17 00:00:00 2001 From: Nicolas Roche Date: Mon, 5 Aug 2024 17:16:41 +0200 Subject: [PATCH 1/2] Ensure file:// urls are generated correctly on Windows --- src/e3/python/pypi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/e3/python/pypi.py b/src/e3/python/pypi.py index ea42dc9f..227150bb 100644 --- a/src/e3/python/pypi.py +++ b/src/e3/python/pypi.py @@ -610,7 +610,7 @@ def add_wheel(self, filename: str) -> None: name = os.path.basename(filename)[:-4].split("-")[0] self.pypi.cache[canonicalize_name(name)] = [ PyPILink( - url=f"file://{os.path.abspath(filename)}", + url=f"file://{os.path.abspath(filename)}".replace("\\", "/"), yanked=None, has_metadata=False, ) From b7e07d9ef17b46622be0a3d0101ba726abe5cabd Mon Sep 17 00:00:00 2001 From: Nicolas Roche Date: Mon, 5 Aug 2024 22:54:03 +0200 Subject: [PATCH 2/2] Review caching mechanism for e3-pypi-closure Ensure environment and requirements for each candidate is not part of the cache. This allow reuse of already computed candidate for various platforms. --- src/e3/python/pypi.py | 126 +++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/src/e3/python/pypi.py b/src/e3/python/pypi.py index 227150bb..f14bde2e 100644 --- a/src/e3/python/pypi.py +++ b/src/e3/python/pypi.py @@ -218,7 +218,6 @@ def fetch_candidates( try: c = PyPICandidate( link=link, - env=env, extras=extras, cache_dir=os.path.join(self.cache_dir, "resources"), ) @@ -263,14 +262,12 @@ class PyPICandidate: def __init__( self, link: PyPILink, - env: dict[str, str], extras: set[str], cache_dir: str, ) -> None: """Initialize a Candidate. :param link: data return by PyPI simple API - :param env: the environment used to evaluate requirements markers :param extras: list of extras that should be included :param cache_dir: cache location in which resources are downloaded """ @@ -308,9 +305,6 @@ def __init__( # Requirements cache self._reqs: None | set[Requirement] = None - # Copy of the environment used to evaluate markers - self.env = dict(env) - @property def is_wheel(self) -> bool: """Check if resource is a wheel.""" @@ -332,74 +326,80 @@ def download(self) -> str: fd.write(answer.content) return download_path - @property - def requirements(self) -> set[Requirement]: + def requirements(self, env: dict[str, str]) -> set[Requirement]: """Return the list of requirements for the package. + :param env: the environment used to evaluate requirements markers :return: a set of Requirement """ - # Check if the requirements have already been computed - if self._reqs is not None: - return self._reqs - - reqs: set[Requirement] = set() - - if self.is_wheel: - # This is a wheel so there is a formal way to get the metadata. - wheel_path = self.download() - reqs |= Wheel(path=wheel_path).requirements + # Make a copy of the env as the function modifies it on the fly + env = dict(env) - elif self.filename.endswith(".tar.gz"): - # This is a .tar.gz archive so we might find some info about the - # requirements either in the egg-info data for older packages or - # as fallback in a requirements.txt file. - path = self.download() - with tarfile.open(name=path, mode="r:gz") as fd: - egg_info = f"{self.filename[:-7]}/{self.name}.egg-info" - egg_info_requires = f"{egg_info}/requires.txt" - requirements_txt = f"{self.filename[:-7]}/requirements.txt" - archive_members = fd.getnames() - - if egg_info in archive_members: - # If we egg-info data without requires.txt it means the package - # has no dependencies. - if egg_info_requires in archive_members: - file_fd = fd.extractfile(egg_info_requires) + # Check if the requirements have already been computed + if self._reqs is None: + self._reqs = set() + + if self.is_wheel: + # This is a wheel so there is a formal way to get the metadata. + wheel_path = self.download() + self._reqs |= Wheel(path=wheel_path).requirements + + elif self.filename.endswith(".tar.gz"): + # This is a .tar.gz archive so we might find some info about the + # requirements either in the egg-info data for older packages or + # as fallback in a requirements.txt file. + path = self.download() + with tarfile.open(name=path, mode="r:gz") as fd: + egg_info = f"{self.filename[:-7]}/{self.name}.egg-info" + egg_info_requires = f"{egg_info}/requires.txt" + requirements_txt = f"{self.filename[:-7]}/requirements.txt" + archive_members = fd.getnames() + + if egg_info in archive_members: + # If we have egg-info data without requires.txt it means the + # package has no dependencies. + if egg_info_requires in archive_members: + file_fd = fd.extractfile(egg_info_requires) + assert file_fd is not None + requires = file_fd.read().decode("utf-8") + self._reqs |= { + Requirement(line.strip()) + for line in requires.splitlines() + } + + elif requirements_txt in archive_members: + # Check if there is a requirements.txt (this is a fallback) + file_fd = fd.extractfile(requirements_txt) assert file_fd is not None requires = file_fd.read().decode("utf-8") - reqs |= { + self._reqs |= { Requirement(line.strip()) for line in requires.splitlines() } - elif requirements_txt in archive_members: - # Check if there is a requirements.txt (this is a fallback) - file_fd = fd.extractfile(requirements_txt) - assert file_fd is not None - requires = file_fd.read().decode("utf-8") - reqs |= { - Requirement(line.strip()) for line in requires.splitlines() - } - - else: - logger.warning(f"Cannot follow dependencies of package {self.name}") + else: + logger.warning( + f"Cannot follow dependencies of package {self.name}" + ) # Once we have the complete list of requirements, use the env to filter - # out requirements not needed for the current configuration. - self._reqs = set() + # out requirements not needed for the current configuration. Don't cache that + # result as it depends on the current environment. + reqs: set[Requirement] = set() + if self.extras: # Special handling for extras. An additional dependencies is added # to the package itself without extras for extra in self.extras: - self.env["extra"] = extra - for r in reqs: - if r.marker is not None and r.marker.evaluate(self.env): - self._reqs.add(r) - self._reqs.add(Requirement(f"{self.name} == {self.version}")) + env["extra"] = extra + for r in self._reqs: + if r.marker is not None and r.marker.evaluate(env): + reqs.add(r) + reqs.add(Requirement(f"{self.name} == {self.version}")) else: - for r in reqs: - if r.marker is None or r.marker.evaluate(self.env): - self._reqs.add(r) - return self._reqs + for r in self._reqs: + if r.marker is None or r.marker.evaluate(env): + reqs.add(r) + return reqs def is_compatible_with_platforms(self, platform_list: list[str]) -> bool: """Check if the package is compatible with a list of platform. @@ -486,7 +486,6 @@ def __init__(self, env: dict[str, str], pypi: PyPI): super().__init__() self.pypi = pypi self.env = env - self.candidate_cache: dict[str, PyPICandidate] = {} def identify(self, requirement_or_candidate: Requirement | PyPICandidate) -> str: """See resolvelib documentation.""" @@ -565,7 +564,7 @@ def is_satisfied_by( def get_dependencies(self, candidate: PyPICandidate) -> Iterable[Requirement]: """See resolvelib documentation.""" - return candidate.requirements + return candidate.requirements(env=self.env) class PyPIClosure: @@ -678,14 +677,15 @@ def requirements_closure(self) -> list[Requirement]: :return: return a list of requirement that can be used as a lock file """ all_reqs = self._requirements_closure() - reqs = [] + reqs = set() for k, v in all_reqs.items(): if len(v) == len(self.platforms): - reqs.append(k) + reqs.add(k) else: for p in v: - reqs.append(Requirement(f'{v}; sys_platform == "{p}"')) - return reqs + sys_platform = get_pip_env(p, self.python3_version)["sys_platform"] + reqs.add(Requirement(f'{k}; sys_platform == "{sys_platform}"')) + return sorted(reqs, key=lambda r: r.name) def __enter__(self) -> PyPIClosure: return self