From d9109a824e5e42481bfaff6d6d8634a161c5f9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sun, 10 Dec 2023 09:02:46 -0800 Subject: [PATCH] First commit (#1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- .github/dependabot.yml | 6 + .github/release.yml | 5 + .github/workflows/check.yml | 91 ++++ .github/workflows/release.yml | 27 + .gitignore | 9 + .pre-commit-config.yaml | 35 ++ .readthedocs.yml | 8 + CODE_OF_CONDUCT.md | 60 +++ LICENSE | 18 + README.md | 9 +- docs/conf.py | 40 ++ docs/index.rst | 20 + pyproject.toml | 116 +++++ src/py_discovery/__init__.py | 21 + src/py_discovery/_builtin.py | 194 +++++++ src/py_discovery/_discover.py | 54 ++ src/py_discovery/_info.py | 712 ++++++++++++++++++++++++++ src/py_discovery/_spec.py | 132 +++++ src/py_discovery/_windows/__init__.py | 49 ++ src/py_discovery/_windows/pep514.py | 243 +++++++++ src/py_discovery/py.typed | 0 tests/conftest.py | 26 + tests/test_discovery.py | 81 +++ tests/test_py_info.py | 444 ++++++++++++++++ tests/test_py_info_exe_based_of.py | 67 +++ tests/test_py_spec.py | 118 +++++ tests/test_version.py | 7 + tests/windows/test_windows.py | 182 +++++++ tests/windows/winreg-mock-values.py | 154 ++++++ tox.ini | 86 ++++ 30 files changed, 3013 insertions(+), 1 deletion(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/release.yml create mode 100644 .github/workflows/check.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .readthedocs.yml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 LICENSE create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 pyproject.toml create mode 100644 src/py_discovery/__init__.py create mode 100644 src/py_discovery/_builtin.py create mode 100644 src/py_discovery/_discover.py create mode 100644 src/py_discovery/_info.py create mode 100644 src/py_discovery/_spec.py create mode 100644 src/py_discovery/_windows/__init__.py create mode 100644 src/py_discovery/_windows/pep514.py create mode 100644 src/py_discovery/py.typed create mode 100644 tests/conftest.py create mode 100644 tests/test_discovery.py create mode 100644 tests/test_py_info.py create mode 100644 tests/test_py_info_exe_based_of.py create mode 100644 tests/test_py_spec.py create mode 100644 tests/test_version.py create mode 100644 tests/windows/test_windows.py create mode 100644 tests/windows/winreg-mock-values.py create mode 100644 tox.ini diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1230149 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..9d1e098 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,5 @@ +changelog: + exclude: + authors: + - dependabot + - pre-commit-ci diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..b85aadf --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,91 @@ +name: check +on: + workflow_dispatch: + push: + branches: "main" + tags-ignore: ["**"] + pull_request: + schedule: + - cron: "0 8 * * *" + +concurrency: + group: check-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: test ${{ matrix.py }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + py: + - "3.12" + - "3.11" + - "3.10" + - "3.9" + - "3.8" + - "3.7" + - "pypy3.10" + - "pypy3.7" + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Setup python for tox + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install tox + run: python -m pip install tox + - name: Setup python for test ${{ matrix.py }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.py }} + - name: Setup test suite + run: tox r -e ${{ matrix.py }} --skip-missing-interpreters false -vv --notest + env: + FORCE_COLOR: "1" + - name: Run test suite + run: tox r -e ${{ matrix.py }} --skip-missing-interpreters false --skip-pkg-install + env: + FORCE_COLOR: "1" + PYTEST_ADDOPTS: "-vv --durations=20" + CI_RUN: "yes" + DIFF_AGAINST: HEAD + + check: + name: tox env ${{ matrix.tox_env }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + tox_env: + - type + - dev + - docs + - pkg_meta + os: + - ubuntu-latest + - windows-latest + exclude: + - { os: windows-latest, tox_env: pkg_meta } # would be the same + - { os: ubuntu-latest, tox_env: docs } # runs on readthedocs.org already + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install tox + run: python -m pip install tox + - name: Run check for ${{ matrix.tox_env }} + run: tox -e ${{ matrix.tox_env }} + env: + UPGRADE_ADVISORY: "yes" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..589f131 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: Release to PyPI +on: + push: + tags: ["*"] + +jobs: + release: + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/pyproject-api + permissions: + id-token: write + steps: + - name: Setup python to build package + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build + run: python -m pip install build + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build package + run: pyproject-build -s -w . -o dist + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.8.11 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f841c9c --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.py[cod] +*.swp +__pycache__ +/src/py_discovery/_version.py +build +dist +*.egg-info +.tox +/.*_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a814303 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.1.7" + hooks: + - id: ruff-format + - id: ruff + args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] + - repo: https://github.com/tox-dev/tox-ini-fmt + rev: "1.3.1" + hooks: + - id: tox-ini-fmt + args: ["-p", "fix"] + - repo: https://github.com/tox-dev/pyproject-fmt + rev: "1.5.3" + hooks: + - id: pyproject-fmt + additional_dependencies: ["tox>=4.11.4"] + - repo: https://github.com/asottile/blacken-docs + rev: 1.16.0 + hooks: + - id: blacken-docs + additional_dependencies: [black==23.11] + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..8629196 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,8 @@ +version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.12" + commands: + - pip install tox + - tox r -e docs -- "${READTHEDOCS_OUTPUT}"/html diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d2f2eac --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,60 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making +participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, +disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, +religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take +appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, +issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the +project or its community. Examples of representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed representative at an online or offline +event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at +tox-dev@python.org. The project team will review and investigate all complaints, and will respond in a way that it deems +appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter +of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent +repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at +[https://www.contributor-covenant.org/version/1/4/code-of-conduct.html][version] + +[homepage]: https://www.contributor-covenant.org/ +[version]: https://www.contributor-covenant.org/version/1/4/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3649823 --- /dev/null +++ b/LICENSE @@ -0,0 +1,18 @@ +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index d2a1ba4..8b7e5c9 100644 --- a/README.md +++ b/README.md @@ -1 +1,8 @@ -# py-discovery \ No newline at end of file +# [`py-discovery`](https://py-discovery.readthedocs.io/en/latest/) + +[![PyPI](https://img.shields.io/pypi/v/py-discovery?style=flat-square)](https://pypi.org/project/py-discovery/) +[![Supported Python +versions](https://img.shields.io/pypi/pyversions/py-discovery.svg)](https://pypi.org/project/py-discovery/) +[![Downloads](https://static.pepy.tech/badge/py-discovery/month)](https://pepy.tech/project/py-discovery) +[![check](https://github.com/tox-dev/py-discovery/actions/workflows/check.yml/badge.svg)](https://github.com/tox-dev/py-discovery/actions/workflows/check.yml) +[![Documentation Status](https://readthedocs.org/projects/py-discovery/badge/?version=latest)](https://py-discovery.readthedocs.io/en/latest/?badge=latest) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..6773e5f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,40 @@ +# noqa: D100 +from __future__ import annotations + +from py_discovery import __version__ + +project = name = "py_discovery" +company = "tox-dev" +copyright = f"{company}" # noqa: A001 +version, release = __version__, __version__.split("+")[0] + +extensions = [ + "sphinx.ext.autosectionlabel", + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx_autodoc_typehints", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", +] +master_doc, source_suffix = "index", ".rst" + +html_theme = "furo" +html_title, html_last_updated_fmt = "py-discovery docs", "%Y-%m-%dT%H:%M:%S" +pygments_style, pygments_dark_style = "sphinx", "monokai" + +autoclass_content, autodoc_typehints = "both", "none" +autodoc_default_options = {"members": True, "member-order": "bysource", "undoc-members": True, "show-inheritance": True} +inheritance_alias = {} + +extlinks = { + "issue": ("https://github.com/tox-dev/py-discovery/issues/%s", "#%s"), + "pull": ("https://github.com/tox-dev/py-discovery/pull/%s", "PR #%s"), + "user": ("https://github.com/%s", "@%s"), +} +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "packaging": ("https://packaging.pypa.io/en/latest", None), +} + +nitpicky = True +nitpick_ignore = [] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..c7ded2c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +``py-discovery`` +================= + +``py-discovery`` aims to abstract away discovering Python interpreters on a user machine. + +API ++++ + +.. currentmodule:: py_discovery + +.. autodata:: __version__ + +.. automodule:: py_discovery + :members: + :undoc-members: + +.. toctree:: + :hidden: + + self diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3f91483 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,116 @@ +[build-system] +build-backend = "hatchling.build" +requires = [ + "hatch-vcs>=0.3", + "hatchling>=1.17.1", +] + +[project] +name = "py-discovery" +description = "API to interact with the python pyproject.toml based projects" +readme.content-type = "text/markdown" +readme.file = "README.md" +keywords = [ + "environments", + "isolated", + "testing", + "virtual", +] +license = "MIT" +maintainers = [{ name = "Bernát Gábor", email = "gaborjbernat@gmail.com" }] +authors = [{ name = "Bernát Gábor", email = "gaborjbernat@gmail.com" }] +requires-python = ">=3.7" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: tox", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Testing", + "Topic :: Utilities", +] +dynamic = [ + "version", +] +dependencies = [ + 'typing-extensions>=4.7.1; python_version < "3.11"', +] +optional-dependencies.docs = [ + "furo>=2023.9.10", + "sphinx<7.2", + "sphinx-autodoc-typehints>=1.25.2", +] +optional-dependencies.testing = [ + "covdefaults>=2.3", + "pytest>=7.4.3", + "pytest-cov>=4.1", + "pytest-mock>=3.11.1", + "setuptools>=68", +] +urls.Homepage = "https://py-discovery.readthedocs.io" +urls.Source = "https://github.com/tox-dev/py-discovery" +urls.Tracker = "https://github.com/tox-dev/py-discovery/issues" + +[tool.hatch] +build.hooks.vcs.version-file = "src/py_discovery/_version.py" +version.source = "vcs" + +[tool.black] +line-length = 120 + +[tool.ruff] +select = ["ALL"] +line-length = 120 +target-version = "py37" +isort = {known-first-party = ["py_discovery"], required-imports = ["from __future__ import annotations"]} +ignore = [ + "INP001", # no implicit namespaces here + "ANN101", # Missing type annotation for `self` in method + "ANN102", # Missing type annotation for `cls` in classmethod" + "ANN401", # Dynamically typed expressions + "D203", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible + "D212", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible + "S104", # Possible binding to all interfaces +] +[tool.ruff.per-file-ignores] +"tests/**/*.py" = [ + "S101", # asserts allowed in tests + "FBT", # don't care about booleans as positional arguments in tests + "INP001", # no implicit namespace + "D", # don't care about documentation in tests + "S603", # `subprocess` call: check for execution of untrusted input + "PLR2004", # Magic value used in comparison, consider replacing with a constant variable +] + +[tool.coverage] +report.fail_under = 88.8 +html.show_contexts = true +html.skip_covered = false +paths.source = [ + "src", + ".tox*/*/lib/python*/site-packages", + ".tox*/pypy*/site-packages", + ".tox/pypy*/lib/pypy*/site-packages", + ".tox*\\*\\Lib\\site-packages", + "*/src", + "*\\src", +] +report.omit = [] +run.parallel = true +run.plugins = ["covdefaults"] + +[tool.mypy] +python_version = "3.7" +show_error_codes = true +strict = true +overrides = [{ module = ["setuptools.*"], ignore_missing_imports = true }] diff --git a/src/py_discovery/__init__.py b/src/py_discovery/__init__.py new file mode 100644 index 0000000..2725a46 --- /dev/null +++ b/src/py_discovery/__init__.py @@ -0,0 +1,21 @@ +"""Python discovery.""" +from __future__ import annotations + +from ._builtin import Builtin, PathPythonInfo, get_interpreter +from ._discover import Discover +from ._info import PythonInfo, VersionInfo +from ._spec import PythonSpec +from ._version import version + +__version__ = version #: version of the package + +__all__ = [ + "PythonInfo", + "VersionInfo", + "PythonSpec", + "Discover", + "Builtin", + "PathPythonInfo", + "get_interpreter", + "__version__", +] diff --git a/src/py_discovery/_builtin.py b/src/py_discovery/_builtin.py new file mode 100644 index 0000000..e0d5cb6 --- /dev/null +++ b/src/py_discovery/_builtin.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import logging +import os +import sys +from typing import TYPE_CHECKING, Iterator, Mapping, MutableMapping + +from ._discover import Discover +from ._info import PythonInfo +from ._spec import PythonSpec + +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + + +class Builtin(Discover): + def __init__(self, options: Namespace) -> None: + super().__init__(options) + self.python_spec = options.python if options.python else [sys.executable] + self.try_first_with = options.try_first_with + + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser) -> None: + parser.add_argument( + "-p", + "--python", + dest="python", + metavar="py", + type=str, + action="append", + default=[], + help="interpreter based on what to create environment (path/identifier) " + "- by default use the interpreter where the tool is installed - first found wins", + ) + parser.add_argument( + "--try-first-with", + dest="try_first_with", + metavar="py_exe", + type=str, + action="append", + default=[], + help="try first these interpreters before starting the discovery", + ) + + def run(self) -> PythonInfo | None: + for python_spec in self.python_spec: + result = get_interpreter(python_spec, self.try_first_with, self._env) + if result is not None: + return result + return None + + def __repr__(self) -> str: + spec = self.python_spec[0] if len(self.python_spec) == 1 else self.python_spec + return f"{self.__class__.__name__} discover of python_spec={spec!r}" + + +def get_interpreter( + key: str, + try_first_with: list[str], + env: MutableMapping[str, str] | None = None, +) -> PythonInfo | None: + spec = PythonSpec.from_string_spec(key) + logging.info("find interpreter for spec %r", spec) + proposed_paths = set() + env = os.environ if env is None else env + for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, env): + if interpreter is None: + continue + lookup_key = interpreter.system_executable, impl_must_match + if lookup_key in proposed_paths: + continue + logging.info("proposed %s", interpreter) + if interpreter.satisfies(spec, impl_must_match): + logging.debug("accepted %s", interpreter) + return interpreter + proposed_paths.add(lookup_key) + return None + + +def propose_interpreters( # noqa: C901, PLR0912 + spec: PythonSpec, + try_first_with: list[str], + env: MutableMapping[str, str] | None = None, +) -> Iterator[tuple[PythonInfo | None, bool]]: + # 0. tries with first + env = os.environ if env is None else env + for py_exe in try_first_with: + path = os.path.abspath(py_exe) # noqa: PTH100 + try: + os.lstat(path) # Windows Store Python does not work with os.path.exists, but does for os.lstat + except OSError: + pass + else: + yield PythonInfo.from_exe(os.path.abspath(path), env=env), True # noqa: PTH100 + + # 1. if it's a path and exists + if spec.path is not None: + try: + os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat + except OSError: + if spec.is_abs: + raise + else: + yield PythonInfo.from_exe(os.path.abspath(spec.path), env=env), True # noqa: PTH100 + if spec.is_abs: + return + else: + # 2. otherwise tries with the current + yield PythonInfo.current_system(), True + + # 3. otherwise fallbacks to platform default logic + if sys.platform == "win32": + from ._windows import propose_interpreters + + for interpreter in propose_interpreters(spec, env): + yield interpreter, True + # finally, find on the path, the path order matters (as the candidates are less easy to control by end user) + paths = get_paths(env) + tested_exes = set() + for pos, path in enumerate(paths): + path_str = str(path) + logging.debug(LazyPathDump(pos, path_str, env)) + for candidate, match in possible_specs(spec): + found = check_path(candidate, path_str) + if found is not None: + exe = os.path.abspath(found) # noqa: PTH100 + if exe not in tested_exes: + tested_exes.add(exe) + got = PathPythonInfo.from_exe(exe, raise_on_error=False, env=env) + if got is not None: + yield got, match + + +def get_paths(env: Mapping[str, str]) -> list[str]: + path = env.get("PATH", None) + if path is None: + if sys.platform == "win32": # pragma: win32 cover + path = os.defpath + else: # pragma: win32 cover + path = os.confstr("CS_PATH") or os.defpath + return [] if not path else [p for p in path.split(os.pathsep) if os.path.exists(p)] # noqa: PTH110 + + +class LazyPathDump: + def __init__(self, pos: int, path: str, env: Mapping[str, str]) -> None: + self.pos = pos + self.path = path + self.env = env + + def __repr__(self) -> str: + content = f"discover PATH[{self.pos}]={self.path}" + if self.env.get("_VIRTUALENV_DEBUG"): # this is the over the board debug + content += " with =>" + for file_name in os.listdir(self.path): + try: + file_path = os.path.join(self.path, file_name) # noqa: PTH118 + if os.path.isdir(file_path) or not os.access(file_path, os.X_OK): # noqa: PTH112 + continue + except OSError: + pass + content += " " + content += file_name + return content + + +def check_path(candidate: str, path: str) -> str | None: + _, ext = os.path.splitext(candidate) # noqa: PTH122 + if sys.platform == "win32" and ext != ".exe": + candidate = f"{candidate}.exe" + if os.path.isfile(candidate): # noqa: PTH113 + return candidate + candidate = os.path.join(path, candidate) # noqa: PTH118 + if os.path.isfile(candidate): # noqa: PTH113 + return candidate + return None + + +def possible_specs(spec: PythonSpec) -> Iterator[tuple[str, bool]]: + # 4. then maybe it's something exact on PATH - if it was a direct lookup implementation no longer counts + if spec.str_spec is not None: + yield spec.str_spec, False + # 5. or from the spec we can deduce a name on path that matches + yield from spec.generate_names() + + +class PathPythonInfo(PythonInfo): + """python info from a path.""" + + +__all__ = [ + "get_interpreter", + "Builtin", + "PathPythonInfo", +] diff --git a/src/py_discovery/_discover.py b/src/py_discovery/_discover.py new file mode 100644 index 0000000..a86bbc3 --- /dev/null +++ b/src/py_discovery/_discover.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + + from py_discovery import PythonInfo + + +class Discover(metaclass=ABCMeta): + """Discover and provide the requested Python interpreter.""" + + @classmethod + def add_parser_arguments(cls, parser: ArgumentParser) -> None: + """ + Add CLI arguments for this discovery mechanisms. + + :param parser: The CLI parser. + """ + raise NotImplementedError + + def __init__(self, options: Namespace) -> None: + """ + Create a new discovery mechanism. + + :param options: The parsed options as defined within the :meth:`add_parser_arguments`. + """ + self._has_run = False + self._interpreter: PythonInfo | None = None + self._env = options.env + + @abstractmethod + def run(self) -> PythonInfo | None: + """ + Discovers an interpreter. + + :return: The interpreter ready to use for virtual environment creation + """ + raise NotImplementedError + + @property + def interpreter(self) -> PythonInfo | None: + """:return: the interpreter as returned by the :meth:`run`, cached""" + if self._has_run is False: + self._interpreter = self.run() + self._has_run = True + return self._interpreter + + +__all__ = [ + "Discover", +] diff --git a/src/py_discovery/_info.py b/src/py_discovery/_info.py new file mode 100644 index 0000000..391a021 --- /dev/null +++ b/src/py_discovery/_info.py @@ -0,0 +1,712 @@ +""" +The PythonInfo contains information about a concrete instance of a Python interpreter. + +Note: this file is also used to query target interpreters, so can only use standard library methods +""" + +from __future__ import annotations + +import copy +import json +import logging +import os +import platform +import re +import sys +import sysconfig +import warnings +from collections import OrderedDict, namedtuple +from pathlib import Path +from random import choice +from shlex import quote +from string import ascii_lowercase, ascii_uppercase, digits +from subprocess import PIPE, Popen +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, Any, Iterator, Mapping, MutableMapping + +if TYPE_CHECKING: + from py_discovery import PythonSpec + +_FS_CASE_SENSITIVE = None + + +def fs_is_case_sensitive() -> bool: + global _FS_CASE_SENSITIVE # noqa: PLW0603 + + if _FS_CASE_SENSITIVE is None: + with NamedTemporaryFile(prefix="TmP") as tmp_file: + _FS_CASE_SENSITIVE = not os.path.exists(tmp_file.name.lower()) # noqa: PTH110 + logging.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") + return _FS_CASE_SENSITIVE + + +VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) # noqa: PYI024 + + +def _get_path_extensions() -> list[str]: + return list(OrderedDict.fromkeys(["", *os.environ.get("PATHEXT", "").lower().split(os.pathsep)])) + + +EXTENSIONS = _get_path_extensions() +_CONF_VAR_RE = re.compile(r"\{\w+\}") + + +class PythonInfo: + """Contains information for a Python interpreter.""" + + def __init__(self) -> None: + def abs_path(v: str | None) -> str | None: + # unroll relative elements from path (e.g. ..) + return None if v is None else os.path.abspath(v) # noqa: PTH100 + + # qualifies the python + self.platform = sys.platform + self.implementation = platform.python_implementation() + self.pypy_version_info = ( + tuple(sys.pypy_version_info) # type: ignore[attr-defined] + if self.implementation == "PyPy" + else None + ) + + # this is a tuple in earlier, struct later, unify to our own named tuple + self.version_info = VersionInfo(*sys.version_info) + self.architecture = 64 if sys.maxsize > 2**32 else 32 + + # Used to determine some file names - see `CPython3Windows.python_zip()`. + self.version_nodot = sysconfig.get_config_var("py_version_nodot") + + self.version = sys.version + self.os = os.name + + # information about the prefix - determines python home + self.prefix = abs_path(getattr(sys, "prefix", None)) # prefix we think + self.base_prefix = abs_path(getattr(sys, "base_prefix", None)) # venv + self.real_prefix = abs_path(getattr(sys, "real_prefix", None)) # old virtualenv + + # information about the exec prefix - dynamic stdlib modules + self.base_exec_prefix = abs_path(getattr(sys, "base_exec_prefix", None)) + self.exec_prefix = abs_path(getattr(sys, "exec_prefix", None)) + + self.executable = abs_path(sys.executable) # the executable we were invoked via + self.original_executable = abs_path(self.executable) # the executable as known by the interpreter + self.system_executable = self._fast_get_system_executable() # the executable we are based of (if available) + + try: + __import__("venv") + has = True + except ImportError: + has = False + self.has_venv = has + self.path = sys.path + self.file_system_encoding = sys.getfilesystemencoding() + self.stdout_encoding = getattr(sys.stdout, "encoding", None) + + scheme_names = sysconfig.get_scheme_names() + + if "venv" in scheme_names: + self.sysconfig_scheme = "venv" + self.sysconfig_paths = { + i: sysconfig.get_path(i, expand=False, scheme=self.sysconfig_scheme) for i in sysconfig.get_path_names() + } + # we cannot use distutils at all if "venv" exists, distutils don't know it + self.distutils_install = {} + # debian / ubuntu python 3.10 without `python3-distutils` will report + # mangled `local/bin` / etc. names for the default prefix + # intentionally select `posix_prefix` which is the unaltered posix-like paths + elif sys.version_info[:2] == (3, 10) and "deb_system" in scheme_names: + self.sysconfig_scheme: str | None = "posix_prefix" + self.sysconfig_paths = { + i: sysconfig.get_path(i, expand=False, scheme=self.sysconfig_scheme) for i in sysconfig.get_path_names() + } + # we cannot use distutils at all if "venv" exists, distutils don't know it + self.distutils_install = {} + else: + self.sysconfig_scheme = None # type: ignore[assignment] + self.sysconfig_paths = {i: sysconfig.get_path(i, expand=False) for i in sysconfig.get_path_names()} + self.distutils_install = self._distutils_install().copy() + + # https://bugs.python.org/issue22199 + makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None)) + # a list of content to store from sysconfig + self.sysconfig = { + k: v for k, v in ([("makefile_filename", makefile())] if makefile is not None else []) if k is not None + } + + config_var_keys = set() + for element in self.sysconfig_paths.values(): + for k in _CONF_VAR_RE.findall(element): + config_var_keys.add(k[1:-1]) + config_var_keys.add("PYTHONFRAMEWORK") + + self.sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys} + + confs = { + k: (self.system_prefix if v is not None and v.startswith(self.prefix) else v) + for k, v in self.sysconfig_vars.items() + } + self.system_stdlib = self.sysconfig_path("stdlib", confs) + self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) + self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) + self._creators = None + + def _fast_get_system_executable(self) -> str | None: + """Try to get the system executable by just looking at properties.""" + # if this is a virtual environment + if self.real_prefix or self.base_prefix is not None and self.base_prefix != self.prefix: + if self.real_prefix is None: + # some platforms may set this to help us + base_executable: str | None = getattr(sys, "_base_executable", None) + if base_executable is not None: # noqa: SIM102 # use the saved system executable if present + if sys.executable != base_executable: # we know we're in a virtual environment, cannot be us + if os.path.exists(base_executable): # noqa: PTH110 + return base_executable + # Python may return "python" because it was invoked from the POSIX virtual environment; but some + # installs/distributions do not provide a version-less python binary in the system install + # location (see PEP 394) so try to fall back to a versioned binary. + # + # Gate this to Python 3.11 as `sys._base_executable` path resolution is now relative to + # the home key from `pyvenv.cfg`, which often points to the system installs location. + major, minor = self.version_info.major, self.version_info.minor + if self.os == "posix" and (major, minor) >= (3, 11): + # search relative to the directory of sys._base_executable + base_dir = os.path.dirname(base_executable) # noqa: PTH120 + versions = (f"python{major}", f"python{major}.{minor}") + for base_executable in [os.path.join(base_dir, exe) for exe in versions]: # noqa: PTH118 + if os.path.exists(base_executable): # noqa: PTH110 + return base_executable + return None # in this case, we just can't tell easily without poking around FS and calling them, bail + # if we're not in a virtual environment, this is already a system python, so return the original executable + # note we must choose the original and not the pure executable as shim scripts might throw us off + return self.original_executable + + def install_path(self, key: str) -> str: + result = self.distutils_install.get(key) + if result is None: # use sysconfig if sysconfig_scheme is set or distutils is unavailable + # set prefixes to empty => result is relative from cwd + prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix + config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()} + result = self.sysconfig_path(key, config_var=config_var).lstrip(os.sep) + return result + + @staticmethod + def _distutils_install() -> dict[str, str]: + # use distutils primarily because that's what pip does + # https://github.com/pypa/pip/blob/main/src/pip/_internal/locations.py#L95 + # note here we don't import Distribution directly to allow setuptools to patch it + with warnings.catch_warnings(): # disable warning for PEP-632 + warnings.simplefilter("ignore") + try: + from distutils import dist + from distutils.command.install import SCHEME_KEYS + except ImportError: # if removed or not installed ignore + return {} + + d = dist.Distribution({"script_args": "--no-user-cfg"}) # conf files not parsed so they do not hijack paths + if hasattr(sys, "_framework"): + sys._framework = None # disable macOS static paths for framework # noqa: SLF001 + + with warnings.catch_warnings(): # disable warning for PEP-632 + warnings.simplefilter("ignore") + i = d.get_command_obj("install", create=True) + assert i is not None # noqa: S101 + + # paths generated are relative to prefix that contains the path sep, this makes it relative + i.prefix = os.sep # type: ignore[attr-defined] + i.finalize_options() + return {key: (getattr(i, f"install_{key}")[1:]).lstrip(os.sep) for key in SCHEME_KEYS} + + @property + def version_str(self) -> str: + return ".".join(str(i) for i in self.version_info[0:3]) + + @property + def version_release_str(self) -> str: + return ".".join(str(i) for i in self.version_info[0:2]) + + @property + def python_name(self) -> str: + version_info = self.version_info + return f"python{version_info.major}.{version_info.minor}" + + @property + def is_old_virtualenv(self) -> bool: + return self.real_prefix is not None + + @property + def is_venv(self) -> bool: + return self.base_prefix is not None + + def sysconfig_path(self, key: str, config_var: dict[str, str] | None = None, sep: str = os.sep) -> str: + pattern = self.sysconfig_paths[key] + if config_var is None: + config_var = self.sysconfig_vars + else: + base = self.sysconfig_vars.copy() + base.update(config_var) + config_var = base + return pattern.format(**config_var).replace("/", sep) + + @property + def system_include(self) -> str: + path = self.sysconfig_path( + "include", + { + k: (self.system_prefix if v is not None and v.startswith(self.prefix) else v) + for k, v in self.sysconfig_vars.items() + }, + ) + if not os.path.exists(path) and self.prefix is not None: # noqa: PTH110 + # some broken packaging doesn't respect the sysconfig, fallback to a distutils path + # the pattern includes the distribution name too at the end, remove that via the parent call + fallback = os.path.join(self.prefix, os.path.dirname(self.install_path("headers"))) # noqa: PTH118, PTH120 + if os.path.exists(fallback): # noqa: PTH110 + path = fallback + return path + + @property + def system_prefix(self) -> str: + res = self.real_prefix or self.base_prefix or self.prefix + assert res is not None # noqa: S101 + return res + + @property + def system_exec_prefix(self) -> str: + res = self.real_prefix or self.base_exec_prefix or self.exec_prefix + assert res is not None # noqa: S101 + return res + + def __unicode__(self) -> str: + return repr(self) + + def __repr__(self) -> str: + return "{}({!r})".format( + self.__class__.__name__, + {k: v for k, v in self.__dict__.items() if not k.startswith("_")}, + ) + + def __str__(self) -> str: + return "{}({})".format( + self.__class__.__name__, + ", ".join( + f"{k}={v}" + for k, v in ( + ("spec", self.spec), + ( + "system" + if self.system_executable is not None and self.system_executable != self.executable + else None, + self.system_executable, + ), + ( + "original" + if self.original_executable not in {self.system_executable, self.executable} + else None, + self.original_executable, + ), + ("exe", self.executable), + ("platform", self.platform), + ("version", repr(self.version)), + ("encoding_fs_io", f"{self.file_system_encoding}-{self.stdout_encoding}"), + ) + if k is not None + ), + ) + + @property + def spec(self) -> str: + return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture) + + def satisfies(self, spec: PythonSpec, impl_must_match: bool) -> bool: # noqa: C901, FBT001 + """Check if a given specification can be satisfied by the python interpreter instance.""" + if spec.path: + if self.executable == os.path.abspath(spec.path): # noqa: PTH100 + return True # if the path is a our own executable path we're done + if not spec.is_abs: + # if path set, and is not our original executable name, this does not match + assert self.original_executable is not None # noqa: S101 + basename = os.path.basename(self.original_executable) # noqa: PTH119 + spec_path = spec.path + if sys.platform == "win32": + basename, suffix = os.path.splitext(basename) # noqa: PTH122 + if spec_path.endswith(suffix): + spec_path = spec_path[: -len(suffix)] + if basename != spec_path: + return False + + if ( + impl_must_match + and spec.implementation is not None + and spec.implementation.lower() != self.implementation.lower() + ): + return False + + if spec.architecture is not None and spec.architecture != self.architecture: + return False + + for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)): + if req is not None and our is not None and our != req: + return False + return True + + _current_system = None + _current = None + + @classmethod + def current(cls) -> PythonInfo: + """ + Locate the current host interpreter information. + + This might be different than what we run into in case the host python has been upgraded from underneath us. + """ + if cls._current is None: + cls._current = cls.from_exe(sys.executable, raise_on_error=True, resolve_to_host=False) + assert cls._current is not None # noqa: S101 + return cls._current + + @classmethod + def current_system(cls) -> PythonInfo: + """ + Locate the current host interpreter information. + + This might be different than what we run into in case the host python has been upgraded from underneath us. + """ + if cls._current_system is None: + cls._current_system = cls.from_exe(sys.executable, raise_on_error=True, resolve_to_host=True) + assert cls._current_system is not None # noqa: S101 + return cls._current_system + + def _to_json(self) -> str: + # don't save calculated paths, as these are non-primitive types + return json.dumps(self._to_dict(), indent=2) + + def _to_dict(self) -> dict[str, Any]: + data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)} + + data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary + return data + + @classmethod + def from_exe( + cls, + exe: str, + *, + raise_on_error: bool = True, + resolve_to_host: bool = True, + env: MutableMapping[str, str] | None = None, + ) -> PythonInfo | None: + """Given a path to an executable, get the python information.""" + # this method is not used by itself, so here and called functions can import stuff locally + + env = os.environ if env is None else env + proposed = cls._from_exe(exe, env=env, raise_on_error=raise_on_error) + + if isinstance(proposed, PythonInfo) and resolve_to_host: + try: + proposed = proposed._resolve_to_system(proposed) # noqa: SLF001 + except Exception as exception: # noqa: BLE001 + if raise_on_error: + raise + logging.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) + proposed = None + return proposed + + @classmethod + def _from_exe( + cls, + exe: str, + env: MutableMapping[str, str] | None = None, + raise_on_error: bool = True, # noqa: FBT001, FBT002 + ) -> PythonInfo | None: + env = os.environ if env is None else env + outcome = _run_subprocess(cls, exe, env) + if isinstance(outcome, Exception): + if raise_on_error: + raise outcome + logging.info("%s", outcome) + return None + outcome.executable = exe + return outcome + + @classmethod + def _from_json(cls, payload: str) -> PythonInfo: + # the dictionary unroll here is to protect against pypy bug of interpreter crashing + raw = json.loads(payload) + return cls._from_dict(raw.copy()) + + @classmethod + def _from_dict(cls, data: dict[str, Any]) -> PythonInfo: + data["version_info"] = VersionInfo(**data["version_info"]) # restore this to a named tuple structure + result = cls() + result.__dict__ = data.copy() + return result + + @classmethod + def _resolve_to_system(cls, target: PythonInfo) -> PythonInfo: + start_executable = target.executable + prefixes: OrderedDict[str, PythonInfo] = OrderedDict() + while target.system_executable is None: + prefix = target.real_prefix or target.base_prefix or target.prefix + assert prefix is not None # noqa: S101 + if prefix in prefixes: + if len(prefixes) == 1: + # if we're linking back to ourselves, accept ourselves with a WARNING + logging.info("%r links back to itself via prefixes", target) + target.system_executable = target.executable + break + for at, (p, t) in enumerate(prefixes.items(), start=1): + logging.error("%d: prefix=%s, info=%r", at, p, t) + logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) + msg = "prefixes are causing a circle {}".format("|".join(prefixes.keys())) + raise RuntimeError(msg) + prefixes[prefix] = target + target = target.discover_exe(prefix=prefix, exact=False) + if target.executable != target.system_executable and target.system_executable is not None: + outcome = cls.from_exe(target.system_executable) + if outcome is None: + msg = "failed to resolve to system executable" + raise RuntimeError(msg) + target = outcome + target.executable = start_executable + return target + + _cache_exe_discovery: dict[tuple[str, bool], PythonInfo] = {} # noqa: RUF012 + + def discover_exe(self, prefix: str, exact: bool = True, env: MutableMapping[str, str] | None = None) -> PythonInfo: # noqa: FBT001, FBT002 + key = prefix, exact + if key in self._cache_exe_discovery and prefix: + logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) + return self._cache_exe_discovery[key] + logging.debug("discover exe for %s in %s", self, prefix) + # we don't know explicitly here, do some guess work - our executable name should tell + possible_names = self._find_possible_exe_names() + possible_folders = self._find_possible_folders(prefix) + discovered: list[PythonInfo] = [] + not_none_env = os.environ if env is None else env + for folder in possible_folders: + for name in possible_names: + info = self._check_exe(folder, name, exact, discovered, not_none_env) + if info is not None: + self._cache_exe_discovery[key] = info + return info + if exact is False and discovered: + info = self._select_most_likely(discovered, self) + folders = os.pathsep.join(possible_folders) + self._cache_exe_discovery[key] = info + logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) + return info + msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) + raise RuntimeError(msg) + + def _check_exe( # noqa: PLR0913 + self, + folder: str, + name: str, + exact: bool, # noqa: FBT001 + discovered: list[PythonInfo], + env: MutableMapping[str, str], + ) -> PythonInfo | None: + exe_path = os.path.join(folder, name) # noqa: PTH118 + if not os.path.exists(exe_path): # noqa: PTH110 + return None + info = self.from_exe(exe_path, resolve_to_host=False, raise_on_error=False, env=env) + if info is None: # ignore if for some reason we can't query + return None + for item in ["implementation", "architecture", "version_info"]: + found = getattr(info, item) + searched = getattr(self, item) + if found != searched: + if item == "version_info": + found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) + executable = info.executable + logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) + if exact is False: + discovered.append(info) + break + else: + return info + return None + + @staticmethod + def _select_most_likely(discovered: list[PythonInfo], target: PythonInfo) -> PythonInfo: + # no exact match found, start relaxing our requirements then to facilitate system package upgrades that + # could cause this (when using copy strategy of the host python) + def sort_by(info: PythonInfo) -> int: + # we need to set up some priority of traits, this is as follows: + # implementation, major, minor, micro, architecture, tag, serial + matches = [ + info.implementation == target.implementation, + info.version_info.major == target.version_info.major, + info.version_info.minor == target.version_info.minor, + info.architecture == target.architecture, + info.version_info.micro == target.version_info.micro, + info.version_info.releaselevel == target.version_info.releaselevel, + info.version_info.serial == target.version_info.serial, + ] + return sum((1 << pos if match else 0) for pos, match in enumerate(reversed(matches))) + + sorted_discovered = sorted(discovered, key=sort_by, reverse=True) # sort by priority in decreasing order + return sorted_discovered[0] + + def _find_possible_folders(self, inside_folder: str) -> list[str]: + candidate_folder: OrderedDict[str, None] = OrderedDict() + executables: OrderedDict[str, None] = OrderedDict() + assert self.executable is not None # noqa: S101 + executables[os.path.realpath(self.executable)] = None + executables[self.executable] = None + assert self.original_executable is not None # noqa: S101 + executables[os.path.realpath(self.original_executable)] = None + executables[self.original_executable] = None + for exe in executables: + base = os.path.dirname(exe) # noqa: PTH120 + # following path pattern of the current + assert self.prefix is not None # noqa: S101 + if base.startswith(self.prefix): + relative = base[len(self.prefix) :] + candidate_folder[f"{inside_folder}{relative}"] = None + + # or at root level + candidate_folder[inside_folder] = None + return [i for i in candidate_folder if os.path.exists(i)] # noqa: PTH110 + + def _find_possible_exe_names(self) -> list[str]: + name_candidate: OrderedDict[str, None] = OrderedDict() + for name in self._possible_base(): + for at in (3, 2, 1, 0): + version = ".".join(str(i) for i in self.version_info[:at]) + for arch in [f"-{self.architecture}", ""]: + for ext in EXTENSIONS: + candidate = f"{name}{version}{arch}{ext}" + name_candidate[candidate] = None + return list(name_candidate.keys()) + + def _possible_base(self) -> Iterator[str]: + possible_base: OrderedDict[str, None] = OrderedDict() + assert self.executable is not None # noqa: S101 + basename = os.path.splitext(os.path.basename(self.executable))[0].rstrip(digits) # noqa: PTH119, PTH122 + possible_base[basename] = None + possible_base[self.implementation] = None + # python is always the final option as in practice is used by multiple implementation as exe name + if "python" in possible_base: + del possible_base["python"] + possible_base["python"] = None + for base in possible_base: + lower = base.lower() + yield lower + + if fs_is_case_sensitive(): + if base != lower: + yield base + upper = base.upper() + if upper != base: + yield upper + + +_COOKIE_LENGTH: int = 32 + + +def _gen_cookie() -> str: + return "".join(choice(f"{ascii_lowercase}{ascii_uppercase}{digits}") for _ in range(_COOKIE_LENGTH)) # noqa: S311 + + +def _run_subprocess(cls: type[PythonInfo], exe: str, env: MutableMapping[str, str]) -> PythonInfo | RuntimeError: + py_info_script = Path(os.path.abspath(__file__)).parent / "_info.py" # noqa: PTH100 + # Cookies allow splitting the serialized stdout output generated by the script collecting the info from the output + # generated by something else. + # The right way to deal with it is to create an anonymous pipe and pass its descriptor to the child and output to + # it. + # However, AFAIK all of them are either not cross-platform or too big to implement and are not in the stdlib; so the + # easiest and the shortest way I could mind is just using the cookies. + # We generate pseudorandom cookies because it is easy to implement and avoids breakage from outputting modules + # source code, i.e., by debug output libraries. + # We reverse the cookies to avoid breakages resulting from variable values appearing in debug output. + + start_cookie = _gen_cookie() + end_cookie = _gen_cookie() + cmd = [exe, str(py_info_script), start_cookie, end_cookie] + # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 + env = copy.copy(env) + env.pop("__PYVENV_LAUNCHER__", None) + logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) + try: + process = Popen( + cmd, # noqa: S603 + universal_newlines=True, + stdin=PIPE, + stderr=PIPE, + stdout=PIPE, + env=env, + encoding="utf-8", + ) + out, err = process.communicate() + code = process.returncode + except OSError as os_error: + out, err, code = "", os_error.strerror, os_error.errno + if code == 0: + out_starts = out.find(start_cookie[::-1]) + + if out_starts > -1: + pre_cookie = out[:out_starts] + + if pre_cookie: + sys.stdout.write(pre_cookie) + + out = out[out_starts + _COOKIE_LENGTH :] + + out_ends = out.find(end_cookie[::-1]) + + if out_ends > -1: + post_cookie = out[out_ends + _COOKIE_LENGTH :] + + if post_cookie: + sys.stdout.write(post_cookie) + + out = out[:out_ends] + + result = cls._from_json(out) + result.executable = exe # keep the original executable as this may contain initialization code + return result + + msg = f"{exe} with code {code}{f' out: {out!r}' if out else ''}{f' err: {err!r}' if err else ''}" + return RuntimeError(f"failed to query {msg}") + + +class LogCmd: + def __init__(self, cmd: list[str], env: Mapping[str, str] | None = None) -> None: + self.cmd = cmd + self.env = env + + def __repr__(self) -> str: + cmd_repr = " ".join(quote(str(c)) for c in self.cmd) + if self.env is not None: + cmd_repr = f"{cmd_repr} env of {self.env!r}" + return cmd_repr + + +__all__ = [ + "PythonInfo", + "VersionInfo", + "fs_is_case_sensitive", + "EXTENSIONS", +] + + +def _run() -> None: + """Dump a JSON representation of the current python.""" + argv = sys.argv[1:] + if len(argv) >= 1: + start_cookie = argv[0] + argv = argv[1:] + else: + start_cookie = "" + if len(argv) >= 1: + end_cookie = argv[0] + argv = argv[1:] + else: + end_cookie = "" + sys.argv = sys.argv[:1] + argv + info = PythonInfo()._to_json() # noqa: SLF001 + sys.stdout.write("".join((start_cookie[::-1], info, end_cookie[::-1]))) + + +if __name__ == "__main__": + _run() diff --git a/src/py_discovery/_spec.py b/src/py_discovery/_spec.py new file mode 100644 index 0000000..05a67a5 --- /dev/null +++ b/src/py_discovery/_spec.py @@ -0,0 +1,132 @@ +"""A Python specification is an abstract requirement definition of an interpreter.""" + +from __future__ import annotations + +import os +import re +from collections import OrderedDict +from typing import Iterator, Tuple, cast + +from py_discovery._info import fs_is_case_sensitive + +PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?:-(?P32|64))?$") + + +class PythonSpec: + """Contains specification about a Python Interpreter.""" + + def __init__( # noqa: PLR0913 + self, + str_spec: str | None, + implementation: str | None, + major: int | None, + minor: int | None, + micro: int | None, + architecture: int | None, + path: str | None, + ) -> None: + self.str_spec = str_spec + self.implementation = implementation + self.major = major + self.minor = minor + self.micro = micro + self.architecture = architecture + self.path = path + + @classmethod + def from_string_spec(cls, string_spec: str) -> PythonSpec: # noqa: C901, PLR0912 + impl, major, minor, micro, arch, path = None, None, None, None, None, None + if os.path.isabs(string_spec): # noqa: PTH117 + path = string_spec + else: + ok = False + match = re.match(PATTERN, string_spec) + if match: + + def _int_or_none(val: str | None) -> int | None: + return None if val is None else int(val) + + try: + groups = match.groupdict() + version = groups["version"] + if version is not None: + versions = tuple(int(i) for i in version.split(".") if i) + if len(versions) > 3: # noqa: PLR2004 + raise ValueError # noqa: TRY301 + if len(versions) == 3: # noqa: PLR2004 + major, minor, micro = versions + elif len(versions) == 2: # noqa: PLR2004 + major, minor = versions + elif len(versions) == 1: + version_data = versions[0] + major = int(str(version_data)[0]) # first digit major + if version_data > 9: # noqa: PLR2004 + minor = int(str(version_data)[1:]) + ok = True + except ValueError: + pass + else: + impl = groups["impl"] + if impl in {"py", "python"}: + impl = None + arch = _int_or_none(groups["arch"]) + + if not ok: + path = string_spec + + return cls(string_spec, impl, major, minor, micro, arch, path) + + def generate_names(self) -> Iterator[tuple[str, bool]]: + impls = OrderedDict() + if self.implementation: + # first, consider implementation as it is + impls[self.implementation] = False + if fs_is_case_sensitive(): + # for case-sensitive file systems, consider lower and upper case versions too + # trivia: MacBooks and all pre-2018 Windows-es were case-insensitive by default + impls[self.implementation.lower()] = False + impls[self.implementation.upper()] = False + impls["python"] = True # finally, consider python as alias; implementation must match now + version = self.major, self.minor, self.micro + try: + not_none_version: tuple[int, ...] = version[: version.index(None)] # type: ignore[assignment] + except ValueError: + not_none_version = cast(Tuple[int, ...], version) + + for impl, match in impls.items(): + for at in range(len(not_none_version), -1, -1): + cur_ver = not_none_version[0:at] + spec = f"{impl}{'.'.join(str(i) for i in cur_ver)}" + yield spec, match + + @property + def is_abs(self) -> bool: + return self.path is not None and os.path.isabs(self.path) # noqa: PTH117 + + def satisfies(self, spec: PythonSpec) -> bool: + """Call when there's a candidate metadata spec to see if compatible - e.g., PEP-514 on Windows.""" + if spec.is_abs and self.is_abs and self.path != spec.path: + return False + if ( + spec.implementation is not None + and self.implementation is not None + and spec.implementation.lower() != self.implementation.lower() + ): + return False + if spec.architecture is not None and spec.architecture != self.architecture: + return False + + for our, req in zip((self.major, self.minor, self.micro), (spec.major, spec.minor, spec.micro)): + if req is not None and our is not None and our != req: + return False + return True + + def __repr__(self) -> str: + name = type(self).__name__ + params = "implementation", "major", "minor", "micro", "architecture", "path" + return f"{name}({', '.join(f'{k}={getattr(self, k)}' for k in params if getattr(self, k) is not None)})" + + +__all__ = [ + "PythonSpec", +] diff --git a/src/py_discovery/_windows/__init__.py b/src/py_discovery/_windows/__init__.py new file mode 100644 index 0000000..5bf5968 --- /dev/null +++ b/src/py_discovery/_windows/__init__.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from typing import Iterator, MutableMapping + +from py_discovery._info import PythonInfo +from py_discovery._spec import PythonSpec + +from .pep514 import discover_pythons + +# Map of well-known organizations (as per PEP 514 Company Windows Registry key part) versus Python implementation +_IMPLEMENTATION_BY_ORG = { + "ContinuumAnalytics": "CPython", + "PythonCore": "CPython", +} + + +class Pep514PythonInfo(PythonInfo): + """A Python information acquired from PEP-514.""" + + +def propose_interpreters(spec: PythonSpec, env: MutableMapping[str, str]) -> Iterator[PythonInfo]: + # see if PEP-514 entries are good + + # start with higher python versions in an effort to use the latest version available + # and prefer PythonCore over conda pythons (as virtualenv is mostly used by non conda tools) + existing = list(discover_pythons()) + existing.sort( + key=lambda i: (*tuple(-1 if j is None else j for j in i[1:4]), 1 if i[0] == "PythonCore" else 0), + reverse=True, + ) + + for name, major, minor, arch, exe, _ in existing: + # Map the well-known/most common organizations to a Python implementation, use the org name as a fallback for + # backwards compatibility. + implementation = _IMPLEMENTATION_BY_ORG.get(name, name) + + # Pre-filtering based on Windows Registry metadata, for CPython only + skip_pre_filter = implementation.lower() != "cpython" + registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe) + if skip_pre_filter or registry_spec.satisfies(spec): + interpreter = Pep514PythonInfo.from_exe(exe, raise_on_error=False, resolve_to_host=True, env=env) + if interpreter is not None and interpreter.satisfies(spec, impl_must_match=True): + yield interpreter # Final filtering/matching using interpreter metadata + + +__all__ = [ + "Pep514PythonInfo", + "propose_interpreters", +] diff --git a/src/py_discovery/_windows/pep514.py b/src/py_discovery/_windows/pep514.py new file mode 100644 index 0000000..0e9968e --- /dev/null +++ b/src/py_discovery/_windows/pep514.py @@ -0,0 +1,243 @@ +"""Implement https://www.python.org/dev/peps/pep-0514/ to discover interpreters - Windows only.""" + +from __future__ import annotations + +import os +import re +import sys +from logging import basicConfig, getLogger +from typing import TYPE_CHECKING, Any, Iterator, Union, cast + +if TYPE_CHECKING: + from types import TracebackType + +if sys.platform == "win32": # pragma: win32 cover + from winreg import ( + HKEY_CURRENT_USER, + HKEY_LOCAL_MACHINE, + KEY_READ, + KEY_WOW64_32KEY, + KEY_WOW64_64KEY, + EnumKey, + HKEYType, + OpenKeyEx, + QueryValueEx, + ) + + +else: # pragma: win32 no cover + HKEY_CURRENT_USER = 0 + HKEY_LOCAL_MACHINE = 1 + KEY_READ = 131097 + KEY_WOW64_32KEY = 512 + KEY_WOW64_64KEY = 256 + + class HKEYType: + def __bool__(self) -> bool: + return True + + def __int__(self) -> int: + return 1 + + def __enter__(self) -> HKEYType: # noqa: PYI034 + return HKEYType() + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: + ... + + def EnumKey(__key: _KeyType, __index: int) -> str: # noqa: N802 + return "" + + def OpenKeyEx( # noqa: N802 + key: _KeyType, # noqa: ARG001 + sub_key: str, # noqa: ARG001 + reserved: int = 0, # noqa: ARG001 + access: int = 131097, # noqa: ARG001 + ) -> HKEYType: + return HKEYType() + + def QueryValueEx(__key: HKEYType, __name: str) -> tuple[Any, int]: # noqa: N802 + return "", 0 + + +_KeyType = Union[HKEYType, int] +LOGGER = getLogger(__name__) + + +def enum_keys(key: _KeyType) -> Iterator[str]: + at = 0 + while True: + try: + yield EnumKey(key, at) + except OSError: + break + at += 1 + + +def get_value(key: HKEYType, value_name: str) -> str | None: + try: + return cast(str, QueryValueEx(key, value_name)[0]) + except OSError: + return None + + +def discover_pythons() -> Iterator[tuple[str, int, int | None, int, str, str | None]]: + for hive, hive_name, key, flags, default_arch in [ + (HKEY_CURRENT_USER, "HKEY_CURRENT_USER", r"Software\Python", 0, 64), + (HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE", r"Software\Python", KEY_WOW64_64KEY, 64), + (HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE", r"Software\Python", KEY_WOW64_32KEY, 32), + ]: + yield from process_set(hive, hive_name, key, flags, default_arch) + + +def process_set( + hive: int, + hive_name: str, + key: str, + flags: int, + default_arch: int, +) -> Iterator[tuple[str, int, int | None, int, str, str | None]]: + try: + with OpenKeyEx(hive, key, 0, KEY_READ | flags) as root_key: + for company in enum_keys(root_key): + if company == "PyLauncher": # reserved + continue + yield from process_company(hive_name, company, root_key, default_arch) + except OSError: + pass + + +def process_company( + hive_name: str, + company: str, + root_key: _KeyType, + default_arch: int, +) -> Iterator[tuple[str, int, int | None, int, str, str | None]]: + with OpenKeyEx(root_key, company) as company_key: + for tag in enum_keys(company_key): + spec = process_tag(hive_name, company, company_key, tag, default_arch) + if spec is not None: + yield spec + + +def process_tag( + hive_name: str, + company: str, + company_key: HKEYType, + tag: str, + default_arch: int, +) -> tuple[str, int, int | None, int, str, str | None] | None: + with OpenKeyEx(company_key, tag) as tag_key: + version = load_version_data(hive_name, company, tag, tag_key) + if version is not None: # if failed to get version bail + major, minor, _ = version + arch = load_arch_data(hive_name, company, tag, tag_key, default_arch) + if arch is not None: + exe_data = load_exe(hive_name, company, company_key, tag) + if exe_data is not None: + exe, args = exe_data + return company, major, minor, arch, exe, args + return None + return None + return None + + +def load_exe(hive_name: str, company: str, company_key: HKEYType, tag: str) -> tuple[str, str | None] | None: + key_path = f"{hive_name}/{company}/{tag}" + try: + with OpenKeyEx(company_key, rf"{tag}\InstallPath") as ip_key, ip_key: + exe = get_value(ip_key, "ExecutablePath") + if exe is None: + ip = get_value(ip_key, "") + if ip is None: + msg(key_path, "no ExecutablePath or default for it") + + else: + ip = ip.rstrip("\\") + exe = f"{ip}\\python.exe" + if exe is not None and os.path.exists(exe): # noqa: PTH110 + args = get_value(ip_key, "ExecutableArguments") + return exe, args + msg(key_path, f"could not load exe with value {exe}") + except OSError: + msg(f"{key_path}/InstallPath", "missing") + return None + + +def load_arch_data(hive_name: str, company: str, tag: str, tag_key: HKEYType, default_arch: int) -> int: + arch_str = get_value(tag_key, "SysArchitecture") + if arch_str is not None: + key_path = f"{hive_name}/{company}/{tag}/SysArchitecture" + try: + return parse_arch(arch_str) + except ValueError as sys_arch: + msg(key_path, sys_arch) + return default_arch + + +def parse_arch(arch_str: str) -> int: + if isinstance(arch_str, str): + match = re.match(r"^(\d+)bit$", arch_str) + if match: + return int(next(iter(match.groups()))) + error = f"invalid format {arch_str}" + else: + error = f"arch is not string: {arch_str!r}" + raise ValueError(error) + + +def load_version_data( + hive_name: str, + company: str, + tag: str, + tag_key: HKEYType, +) -> tuple[int, int | None, int | None] | None: + for candidate, key_path in [ + (get_value(tag_key, "SysVersion"), f"{hive_name}/{company}/{tag}/SysVersion"), + (tag, f"{hive_name}/{company}/{tag}"), + ]: + if candidate is not None: + try: + return parse_version(candidate) + except ValueError as sys_version: + msg(key_path, sys_version) + return None + + +def parse_version(version_str: str) -> tuple[int, int | None, int | None]: + if isinstance(version_str, str): + match = re.match(r"^(\d+)(?:\.(\d+))?(?:\.(\d+))?$", version_str) + if match: + return tuple(int(i) if i is not None else None for i in match.groups()) # type: ignore[return-value] + error = f"invalid format {version_str}" + else: + error = f"version is not string: {version_str!r}" + raise ValueError(error) + + +def msg(path: str, what: str | ValueError) -> None: + LOGGER.warning("PEP-514 violation in Windows Registry at %s error: %s", path, what) + + +def _run() -> None: + basicConfig() + interpreters = [repr(spec) for spec in discover_pythons()] + print("\n".join(sorted(interpreters))) # noqa: T201 + + +__all__ = [ + "HKEY_CURRENT_USER", + "HKEY_LOCAL_MACHINE", + "KEY_READ", + "KEY_WOW64_32KEY", + "KEY_WOW64_64KEY", + "discover_pythons", +] + +if __name__ == "__main__": + _run() diff --git a/src/py_discovery/py.typed b/src/py_discovery/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..49a73c7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import os +import sys +from tempfile import NamedTemporaryFile + +import pytest + + +@pytest.fixture(scope="session") +def _fs_supports_symlink() -> None: + can = False + if hasattr(os, "symlink"): # pragma: no branch + if sys.platform == "win32": # pragma: win32 cover + with NamedTemporaryFile(prefix="TmP") as tmp_file: + temp_dir = os.path.dirname(tmp_file.name) # noqa: PTH120 + dest = os.path.join(temp_dir, f"{tmp_file.name}-{'b'}") # noqa: PTH118 + try: + os.symlink(tmp_file.name, dest) + can = True + except (OSError, NotImplementedError): # pragma: no cover + pass # pragma: no cover + else: # pragma: win32 no cover + can = True + if not can: # pragma: no branch + pytest.skip("No symlink support") # pragma: no cover diff --git a/tests/test_discovery.py b/tests/test_discovery.py new file mode 100644 index 0000000..7d64a3b --- /dev/null +++ b/tests/test_discovery.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import logging +import os +import sys +from argparse import Namespace +from pathlib import Path +from uuid import uuid4 + +import pytest + +from py_discovery import Builtin, PythonInfo, get_interpreter + + +@pytest.mark.usefixtures("_fs_supports_symlink") +@pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) +def test_discovery_via_path( + monkeypatch: pytest.MonkeyPatch, + case: str, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.DEBUG) + current = PythonInfo.current_system() + core = f"somethingVeryCryptic{'.'.join(str(i) for i in current.version_info[0:3])}" + name = "somethingVeryCryptic" + if case == "lower": + name = name.lower() + elif case == "upper": + name = name.upper() + exe_name = f"{name}{current.version_info.major}{'.exe' if sys.platform == 'win32' else ''}" + target = tmp_path / current.install_path("scripts") + target.mkdir(parents=True) + executable = target / exe_name + os.symlink(sys.executable, str(executable)) + pyvenv_cfg = Path(sys.executable).parents[1] / "pyvenv.cfg" + if pyvenv_cfg.exists(): # pragma: no branch + (target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes()) # pragma: no cover + new_path = os.pathsep.join([str(target), *os.environ.get("PATH", "").split(os.pathsep)]) + monkeypatch.setenv("PATH", new_path) + interpreter = get_interpreter(core, []) + + assert interpreter is not None + + +def test_discovery_via_path_not_found(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PATH", str(tmp_path)) + interpreter = get_interpreter(uuid4().hex, []) + assert interpreter is None + + +def test_relative_path(monkeypatch: pytest.MonkeyPatch) -> None: + sys_executable_str = PythonInfo.current_system().system_executable + assert sys_executable_str is not None + sys_executable = Path(sys_executable_str) + cwd = sys_executable.parents[1] + monkeypatch.chdir(str(cwd)) + relative = str(sys_executable.relative_to(cwd)) + result = get_interpreter(relative, []) + assert result is not None + + +def test_discovery_fallback_fail(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.DEBUG) + builtin = Builtin(Namespace(try_first_with=[], python=["magic-one", "magic-two"], env=os.environ)) + + result = builtin.run() + assert result is None + + assert "accepted" not in caplog.text + + +def test_discovery_fallback_ok(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.DEBUG) + builtin = Builtin(Namespace(try_first_with=[], python=["magic-one", sys.executable], env=os.environ)) + + result = builtin.run() + assert result is not None, caplog.text + assert result.executable == sys.executable, caplog.text + + assert "accepted" in caplog.text diff --git a/tests/test_py_info.py b/tests/test_py_info.py new file mode 100644 index 0000000..cf881b0 --- /dev/null +++ b/tests/test_py_info.py @@ -0,0 +1,444 @@ +from __future__ import annotations + +import copy +import functools +import itertools +import json +import logging +import os +import sys +import sysconfig +from pathlib import Path +from platform import python_implementation +from textwrap import dedent +from typing import TYPE_CHECKING, Mapping, NamedTuple, Tuple, cast + +import pytest + +from py_discovery import PythonInfo, PythonSpec, VersionInfo + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + +CURRENT = PythonInfo.current_system() + + +def test_current_as_json() -> None: + result = CURRENT._to_json() # noqa: SLF001 + parsed = json.loads(result) + a, b, c, d, e = sys.version_info + assert parsed["version_info"] == {"major": a, "minor": b, "micro": c, "releaselevel": d, "serial": e} + + +def test_bad_exe_py_info_raise(tmp_path: Path) -> None: + exe = str(tmp_path) + with pytest.raises(RuntimeError) as context: + PythonInfo.from_exe(exe) + msg = str(context.value) + assert "code" in msg + assert exe in msg + + +def test_bad_exe_py_info_no_raise( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + capsys: pytest.CaptureFixture[str], +) -> None: + caplog.set_level(logging.NOTSET) + exe = str(tmp_path) + result = PythonInfo.from_exe(exe, raise_on_error=False) + assert result is None + out, _ = capsys.readouterr() + assert not out + messages = [r.message for r in caplog.records if r.name != "filelock"] + assert len(messages) == 2 + msg = messages[0] + assert "get interpreter info via cmd: " in msg + msg = messages[1] + assert str(exe) in msg + assert "code" in msg + + +@pytest.mark.parametrize( + "spec", + itertools.chain( + [sys.executable], + [ + f"{impl}{'.'.join(str(i) for i in ver)}{arch}" + for impl, ver, arch in itertools.product( + ( + [CURRENT.implementation] + + (["python"] if CURRENT.implementation == "CPython" else []) + + ( + [CURRENT.implementation.lower()] + if CURRENT.implementation != CURRENT.implementation.lower() + else [] + ) + ), + [sys.version_info[0 : i + 1] for i in range(3)], + ["", f"-{CURRENT.architecture}"], + ) + ], + ), +) +def test_satisfy_py_info(spec: str) -> None: + parsed_spec = PythonSpec.from_string_spec(spec) + matches = CURRENT.satisfies(parsed_spec, True) + assert matches is True + + +def test_satisfy_not_arch() -> None: + parsed_spec = PythonSpec.from_string_spec( + f"{CURRENT.implementation}-{64 if CURRENT.architecture == 32 else 32}", + ) + matches = CURRENT.satisfies(parsed_spec, True) + assert matches is False + + +def _generate_not_match_current_interpreter_version() -> list[str]: + result: list[str] = [] + for i in range(3): + ver = cast(Tuple[int, ...], sys.version_info[0 : i + 1]) + for a in range(len(ver)): + for o in [-1, 1]: + temp = list(ver) + temp[a] += o + result.append(".".join(str(i) for i in temp)) + return result + + +_NON_MATCH_VER = _generate_not_match_current_interpreter_version() + + +@pytest.mark.parametrize("spec", _NON_MATCH_VER) +def test_satisfy_not_version(spec: str) -> None: + parsed_spec = PythonSpec.from_string_spec(f"{CURRENT.implementation}{spec}") + matches = CURRENT.satisfies(parsed_spec, True) + assert matches is False + + +class PyInfoMock(NamedTuple): + implementation: str + architecture: int + version_info: VersionInfo + + +@pytest.mark.parametrize( + ("target", "position", "discovered"), + [ + ( + PyInfoMock("CPython", 64, VersionInfo(3, 6, 8, "final", 0)), + 0, + [ + PyInfoMock("CPython", 64, VersionInfo(3, 6, 9, "final", 0)), + PyInfoMock("PyPy", 64, VersionInfo(3, 6, 8, "final", 0)), + ], + ), + ( + PyInfoMock("CPython", 64, VersionInfo(3, 6, 8, "final", 0)), + 0, + [ + PyInfoMock("CPython", 64, VersionInfo(3, 6, 9, "final", 0)), + PyInfoMock("CPython", 32, VersionInfo(3, 6, 9, "final", 0)), + ], + ), + ( + PyInfoMock("CPython", 64, VersionInfo(3, 8, 1, "final", 0)), + 0, + [ + PyInfoMock("CPython", 32, VersionInfo(2, 7, 12, "rc", 2)), + PyInfoMock("PyPy", 64, VersionInfo(3, 8, 1, "final", 0)), + ], + ), + ], +) +def test_system_executable_no_exact_match( # noqa: PLR0913 + target: PyInfoMock, + position: int, + discovered: list[PyInfoMock], + tmp_path: Path, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, +) -> None: + """Here we should fallback to other compatible""" + caplog.set_level(logging.DEBUG) + + def _make_py_info(of: PyInfoMock) -> PythonInfo: + base = copy.deepcopy(CURRENT) + base.implementation = of.implementation + base.version_info = of.version_info + base.architecture = of.architecture + base.system_executable = CURRENT.system_executable + base.executable = CURRENT.system_executable + base.base_executable = str(path) # type: ignore[attr-defined] # we mock it on for the test + return base + + discovered_with_path = {} + names = [] + selected = None + for pos, i in enumerate(discovered): + path = tmp_path / str(pos) + path.write_text("", encoding="utf-8") + py_info = _make_py_info(i) + if pos == position: + selected = py_info + discovered_with_path[str(path)] = py_info + names.append(path.name) + + target_py_info = _make_py_info(target) + mocker.patch.object(target_py_info, "_find_possible_exe_names", return_value=names) + mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) + + def func(k: str, resolve_to_host: bool, raise_on_error: bool, env: Mapping[str, str]) -> PythonInfo: # noqa: ARG001 + return discovered_with_path[k] + + mocker.patch.object(target_py_info, "from_exe", side_effect=func) + target_py_info.real_prefix = str(tmp_path) + + target_py_info.system_executable = None + target_py_info.executable = str(tmp_path) + mapped = target_py_info._resolve_to_system(target_py_info) # noqa: SLF001 + assert mapped.system_executable == CURRENT.system_executable + found = discovered_with_path[mapped.base_executable] # type: ignore[attr-defined] # we set it a few lines above + assert found is selected + + assert caplog.records[0].msg == "discover exe for %s in %s" + for record in caplog.records[1:-1]: + assert record.message.startswith("refused interpreter ") + assert record.levelno == logging.DEBUG + + warn_similar = caplog.records[-1] + assert warn_similar.levelno == logging.DEBUG + assert warn_similar.msg.startswith("no exact match found, chosen most similar") + + +def test_py_info_ignores_distutils_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + raw = f""" + [install] + prefix={tmp_path}{os.sep}prefix + install_purelib={tmp_path}{os.sep}purelib + install_platlib={tmp_path}{os.sep}platlib + install_headers={tmp_path}{os.sep}headers + install_scripts={tmp_path}{os.sep}scripts + install_data={tmp_path}{os.sep}data + """ + (tmp_path / "setup.cfg").write_text(dedent(raw), encoding="utf-8") + monkeypatch.chdir(tmp_path) + py_info = PythonInfo.from_exe(sys.executable) + assert py_info is not None + distutils = py_info.distutils_install + # on newer pythons this is just empty + for key, value in distutils.items(): # pragma: no branch + assert not value.startswith(str(tmp_path)), f"{key}={value}" # pragma: no cover + + +def test_discover_exe_on_path_non_spec_name_match(mocker: MockerFixture) -> None: + suffixed_name = f"python{CURRENT.version_info.major}.{CURRENT.version_info.minor}m" + if sys.platform == "win32": # pragma: win32 cover + assert CURRENT.original_executable is not None + suffixed_name += Path(CURRENT.original_executable).suffix + spec = PythonSpec.from_string_spec(suffixed_name) + assert CURRENT.executable is not None + mocker.patch.object(CURRENT, "original_executable", str(Path(CURRENT.executable).parent / suffixed_name)) + assert CURRENT.satisfies(spec, impl_must_match=True) is True + + +def test_discover_exe_on_path_non_spec_name_not_match(mocker: MockerFixture) -> None: + suffixed_name = f"python{CURRENT.version_info.major}.{CURRENT.version_info.minor}m" + if sys.platform == "win32": # pragma: win32 cover + assert CURRENT.original_executable is not None + suffixed_name += Path(CURRENT.original_executable).suffix + spec = PythonSpec.from_string_spec(suffixed_name) + assert CURRENT.executable is not None + mocker.patch.object( + CURRENT, + "original_executable", + str(Path(CURRENT.executable).parent / f"e{suffixed_name}"), + ) + assert CURRENT.satisfies(spec, impl_must_match=True) is False + + +if python_implementation() != "PyPy": # pragma: pypy no cover + + def test_py_info_setuptools() -> None: + from setuptools.dist import Distribution + + assert Distribution + PythonInfo() + + +if CURRENT.system_executable is None: # pragma: no branch + + def test_py_info_to_system_raises( # pragma: no cover + mocker: MockerFixture, # pragma: no cover + caplog: pytest.LogCaptureFixture, # pragma: no cover + ) -> None: # pragma: no cover + caplog.set_level(logging.DEBUG) # pragma: no cover + mocker.patch.object(PythonInfo, "_find_possible_folders", return_value=[]) # pragma: no cover + result = PythonInfo.from_exe(sys.executable, raise_on_error=False) # pragma: no cover + assert result is None # pragma: no cover + log = caplog.records[-1] # pragma: no cover + assert log.levelno == logging.INFO # pragma: no cover + exe = sys.executable # pragma: no cover + expected = f"ignore {exe} due cannot resolve system due to RuntimeError('failed to detect " # pragma: no cover + assert expected in log.message # pragma: no cover + + +def test_custom_venv_install_scheme_is_prefered(mocker: MockerFixture) -> None: + # The paths in this test are Fedora paths, but we set them for nt as well, so the test also works on Windows, + # despite the actual values are nonsense there. + # Values were simplified to be compatible with all the supported Python versions. + default_scheme = { + "stdlib": "{base}/lib/python{py_version_short}", + "platstdlib": "{platbase}/lib/python{py_version_short}", + "purelib": "{base}/local/lib/python{py_version_short}/site-packages", + "platlib": "{platbase}/local/lib/python{py_version_short}/site-packages", + "include": "{base}/include/python{py_version_short}", + "platinclude": "{platbase}/include/python{py_version_short}", + "scripts": "{base}/local/bin", + "data": "{base}/local", + } + venv_scheme = {key: path.replace("local", "") for key, path in default_scheme.items()} + sysconfig_install_schemes = { + "posix_prefix": default_scheme, + "nt": default_scheme, + "pypy": default_scheme, + "pypy_nt": default_scheme, + "venv": venv_scheme, + } + if getattr(sysconfig, "get_preferred_scheme", None): # pragma: no branch + # define the prefix as sysconfig.get_preferred_scheme did before 3.11 + sysconfig_install_schemes["nt" if os.name == "nt" else "posix_prefix"] = default_scheme # pragma: no cover + + # On Python < 3.10, the distutils schemes are not derived from sysconfig schemes, so we mock them as well to assert + # the custom "venv" install scheme has priority + distutils_scheme = { + "purelib": "$base/local/lib/python$py_version_short/site-packages", + "platlib": "$platbase/local/lib/python$py_version_short/site-packages", + "headers": "$base/include/python$py_version_short/$dist_name", + "scripts": "$base/local/bin", + "data": "$base/local", + } + distutils_schemes = { + "unix_prefix": distutils_scheme, + "nt": distutils_scheme, + } + + # We need to mock distutils first, so they don't see the mocked sysconfig, + # if imported for the first time. + # That can happen if the actual interpreter has the "venv" INSTALL_SCHEME + # and hence this is the first time we are touching distutils in this process. + # If distutils saw our mocked sysconfig INSTALL_SCHEMES, we would need + # to define all install schemes. + mocker.patch("distutils.command.install.INSTALL_SCHEMES", distutils_schemes) + mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes) + + pyinfo = PythonInfo() + pyver = f"{pyinfo.version_info.major}.{pyinfo.version_info.minor}" + assert pyinfo.install_path("scripts") == "bin" + assert pyinfo.install_path("purelib").replace(os.sep, "/") == f"lib/python{pyver}/site-packages" + + +if sys.version_info[:2] >= (3, 11): # pragma: >=3.11 cover # pytest.skip doesn't support covdefaults + if sys.platform != "win32": # pragma: win32 no cover # pytest.skip doesn't support covdefaults + + def test_fallback_existent_system_executable(mocker: MockerFixture) -> None: + current = PythonInfo() + # Posix may execute a "python" out of a venv but try to set the base_executable + # to "python" out of the system installation path. PEP 394 informs distributions + # that "python" is not required and the standard `make install` does not provide one + + # Falsify some data to look like we're in a venv + current.prefix = current.exec_prefix = "/tmp/tmp.izZNCyINRj/venv" # noqa: S108 + exe = os.path.join(current.prefix, "bin/python") # noqa: PTH118 + current.executable = current.original_executable = exe + + # Since we don't know if the distribution we're on provides python, use a binary that should not exist + assert current.system_executable is not None + mocker.patch.object( + sys, + "_base_executable", + os.path.join(os.path.dirname(current.system_executable), "idontexist"), # noqa: PTH118,PTH120 + ) + mocker.patch.object(sys, "executable", current.executable) + + # ensure it falls back to an alternate binary name that exists + current._fast_get_system_executable() # noqa: SLF001 + assert current.system_executable is not None + assert os.path.basename(current.system_executable) in [ # noqa: PTH119 + f"python{v}" + for v in (current.version_info.major, f"{current.version_info.major}.{current.version_info.minor}") + ] + assert current.system_executable is not None + assert os.path.exists(current.system_executable) # noqa: PTH110 + + +if sys.version_info[:2] == (3, 10): # pragma: ==3.10 cover # pytest.skip doesn't support covdefaults + + def test_uses_posix_prefix_on_debian_3_10_without_venv(mocker: MockerFixture) -> None: + # this is taken from ubuntu 22.04 /usr/lib/python3.10/sysconfig.py + sysconfig_install_schemes = { + "posix_prefix": { + "stdlib": "{installed_base}/{platlibdir}/python{py_version_short}", + "platstdlib": "{platbase}/{platlibdir}/python{py_version_short}", + "purelib": "{base}/lib/python{py_version_short}/site-packages", + "platlib": "{platbase}/{platlibdir}/python{py_version_short}/site-packages", + "include": "{installed_base}/include/python{py_version_short}{abiflags}", + "platinclude": "{installed_platbase}/include/python{py_version_short}{abiflags}", + "scripts": "{base}/bin", + "data": "{base}", + }, + "posix_home": { + "stdlib": "{installed_base}/lib/python", + "platstdlib": "{base}/lib/python", + "purelib": "{base}/lib/python", + "platlib": "{base}/lib/python", + "include": "{installed_base}/include/python", + "platinclude": "{installed_base}/include/python", + "scripts": "{base}/bin", + "data": "{base}", + }, + "nt": { + "stdlib": "{installed_base}/Lib", + "platstdlib": "{base}/Lib", + "purelib": "{base}/Lib/site-packages", + "platlib": "{base}/Lib/site-packages", + "include": "{installed_base}/Include", + "platinclude": "{installed_base}/Include", + "scripts": "{base}/Scripts", + "data": "{base}", + }, + "deb_system": { + "stdlib": "{installed_base}/{platlibdir}/python{py_version_short}", + "platstdlib": "{platbase}/{platlibdir}/python{py_version_short}", + "purelib": "{base}/lib/python3/dist-packages", + "platlib": "{platbase}/{platlibdir}/python3/dist-packages", + "include": "{installed_base}/include/python{py_version_short}{abiflags}", + "platinclude": "{installed_platbase}/include/python{py_version_short}{abiflags}", + "scripts": "{base}/bin", + "data": "{base}", + }, + "posix_local": { + "stdlib": "{installed_base}/{platlibdir}/python{py_version_short}", + "platstdlib": "{platbase}/{platlibdir}/python{py_version_short}", + "purelib": "{base}/local/lib/python{py_version_short}/dist-packages", + "platlib": "{platbase}/local/lib/python{py_version_short}/dist-packages", + "include": "{installed_base}/local/include/python{py_version_short}{abiflags}", + "platinclude": "{installed_platbase}/local/include/python{py_version_short}{abiflags}", + "scripts": "{base}/local/bin", + "data": "{base}", + }, + } + # reset the default in case we're on a system which doesn't have this problem + sysconfig_get_path = functools.partial(sysconfig.get_path, scheme="posix_local") + + # make it look like python3-distutils is not available + mocker.patch.dict(sys.modules, {"distutils.command": None}) + mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes) + mocker.patch("sysconfig.get_path", sysconfig_get_path) + mocker.patch("sysconfig.get_default_scheme", return_value="posix_local") + + pyinfo = PythonInfo() + pyver = f"{pyinfo.version_info.major}.{pyinfo.version_info.minor}" + assert pyinfo.install_path("scripts") == "bin" + assert pyinfo.install_path("purelib").replace(os.sep, "/") == f"lib/python{pyver}/site-packages" diff --git a/tests/test_py_info_exe_based_of.py b/tests/test_py_info_exe_based_of.py new file mode 100644 index 0000000..c480d3c --- /dev/null +++ b/tests/test_py_info_exe_based_of.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import logging +import os +import sys +from pathlib import Path + +import pytest + +from py_discovery import PythonInfo +from py_discovery._info import EXTENSIONS, fs_is_case_sensitive + +CURRENT = PythonInfo.current() + + +def test_discover_empty_folder(tmp_path: Path) -> None: + with pytest.raises(RuntimeError): + CURRENT.discover_exe(prefix=str(tmp_path)) + + +BASE = (CURRENT.install_path("scripts"), ".") + + +@pytest.mark.usefixtures("_fs_supports_symlink") +@pytest.mark.parametrize("suffix", sorted({".exe", ".cmd", ""} & set(EXTENSIONS) if sys.platform == "win32" else [""])) +@pytest.mark.parametrize("into", BASE) +@pytest.mark.parametrize("arch", [CURRENT.architecture, ""]) +@pytest.mark.parametrize("version", [".".join(str(i) for i in CURRENT.version_info[0:i]) for i in range(3, 0, -1)]) +@pytest.mark.parametrize("impl", [CURRENT.implementation, "python"]) +def test_discover_ok( # noqa: PLR0913 + tmp_path: Path, + suffix: str, + impl: str, + version: str, + arch: str, + into: str, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.DEBUG) + folder = tmp_path / into + folder.mkdir(parents=True, exist_ok=True) + name = f"{impl}{version}" + if arch: + name += f"-{arch}" + name += suffix + dest = folder / name + assert CURRENT.executable is not None + os.symlink(CURRENT.executable, str(dest)) + pyvenv = Path(CURRENT.executable).parents[1] / "pyvenv.cfg" + if pyvenv.exists(): # pragma: no branch + (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") + inside_folder = str(tmp_path) + base = CURRENT.discover_exe(inside_folder) + found = base.executable + assert found is not None + dest_str = str(dest) + if not fs_is_case_sensitive(): # pragma: no branch + found = found.lower() # pragma: no cover + dest_str = dest_str.lower() # pragma: no cover + assert found == dest_str + assert len(caplog.messages) >= 1, caplog.text + assert "get interpreter info via cmd: " in caplog.text + + dest.rename(dest.parent / (dest.name + "-1")) + CURRENT._cache_exe_discovery.clear() # noqa: SLF001 + with pytest.raises(RuntimeError): + CURRENT.discover_exe(inside_folder) diff --git a/tests/test_py_spec.py b/tests/test_py_spec.py new file mode 100644 index 0000000..c861536 --- /dev/null +++ b/tests/test_py_spec.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import sys +from copy import copy +from typing import TYPE_CHECKING, Tuple, cast + +import pytest + +from py_discovery import PythonSpec + +if TYPE_CHECKING: + from pathlib import Path + + +def test_bad_py_spec() -> None: + text = "python2.3.4.5" + spec = PythonSpec.from_string_spec(text) + assert text in repr(spec) + assert spec.str_spec == text + assert spec.path == text + content = vars(spec) + del content["str_spec"] + del content["path"] + assert all(v is None for v in content.values()) + + +def test_py_spec_first_digit_only_major() -> None: + spec = PythonSpec.from_string_spec("278") + assert spec.major == 2 + assert spec.minor == 78 + + +def test_spec_satisfies_path_ok() -> None: + spec = PythonSpec.from_string_spec(sys.executable) + assert spec.satisfies(spec) is True + + +def test_spec_satisfies_path_nok(tmp_path: Path) -> None: + spec = PythonSpec.from_string_spec(sys.executable) + of = PythonSpec.from_string_spec(str(tmp_path)) + assert spec.satisfies(of) is False + + +def test_spec_satisfies_arch() -> None: + spec_1 = PythonSpec.from_string_spec("python-32") + spec_2 = PythonSpec.from_string_spec("python-64") + + assert spec_1.satisfies(spec_1) is True + assert spec_2.satisfies(spec_1) is False + + +@pytest.mark.parametrize( + ("req", "spec"), + [("py", "python"), ("jython", "jython"), ("CPython", "cpython")], +) +def test_spec_satisfies_implementation_ok(req: str, spec: str) -> None: + spec_1 = PythonSpec.from_string_spec(req) + spec_2 = PythonSpec.from_string_spec(spec) + assert spec_1.satisfies(spec_1) is True + assert spec_2.satisfies(spec_1) is True + + +def test_spec_satisfies_implementation_nok() -> None: + spec_1 = PythonSpec.from_string_spec("cpython") + spec_2 = PythonSpec.from_string_spec("jython") + assert spec_2.satisfies(spec_1) is False + assert spec_1.satisfies(spec_2) is False + + +def _version_satisfies_pairs() -> list[tuple[str, str]]: + target: set[tuple[str, str]] = set() + version = tuple(str(i) for i in sys.version_info[0:3]) + for i in range(len(version) + 1): + req = ".".join(version[0:i]) + for j in range(i + 1): + sat = ".".join(version[0:j]) + # can be satisfied in both directions + target.add((req, sat)) + target.add((sat, req)) + return sorted(target) + + +@pytest.mark.parametrize(("req", "spec"), _version_satisfies_pairs()) +def test_version_satisfies_ok(req: str, spec: str) -> None: + req_spec = PythonSpec.from_string_spec(f"python{req}") + sat_spec = PythonSpec.from_string_spec(f"python{spec}") + assert sat_spec.satisfies(req_spec) is True + + +def _version_not_satisfies_pairs() -> list[tuple[str, str]]: + target: set[tuple[str, str]] = set() + version = tuple(str(i) for i in cast(Tuple[int, ...], sys.version_info[0:3])) + for major in range(len(version)): + req = ".".join(version[0 : major + 1]) + for minor in range(major + 1): + sat_ver = list(cast(Tuple[int, ...], sys.version_info[0 : minor + 1])) + for patch in range(minor + 1): + for o in [1, -1]: + temp = copy(sat_ver) + temp[patch] += o + if temp[patch] >= 0: # pragma: no branch + sat = ".".join(str(i) for i in temp) + target.add((req, sat)) + return sorted(target) + + +@pytest.mark.parametrize(("req", "spec"), _version_not_satisfies_pairs()) +def test_version_satisfies_nok(req: str, spec: str) -> None: + req_spec = PythonSpec.from_string_spec(f"python{req}") + sat_spec = PythonSpec.from_string_spec(f"python{spec}") + assert sat_spec.satisfies(req_spec) is False + + +def test_relative_spec(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + a_relative_path = str((tmp_path / "a" / "b").relative_to(tmp_path)) + spec = PythonSpec.from_string_spec(a_relative_path) + assert spec.path == a_relative_path diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..eaff7c0 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,7 @@ +from __future__ import annotations + + +def test_version() -> None: + from py_discovery import __version__ + + assert __version__ diff --git a/tests/windows/test_windows.py b/tests/windows/test_windows.py new file mode 100644 index 0000000..3610b91 --- /dev/null +++ b/tests/windows/test_windows.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import sys +import textwrap +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any, Iterator, Tuple, cast + +import pytest + +from py_discovery import PythonInfo, PythonSpec, VersionInfo +from py_discovery._windows import discover_pythons, pep514, propose_interpreters # type: ignore[attr-defined] + +if TYPE_CHECKING: + from types import TracebackType + + from pytest_mock import MockerFixture + + if sys.version_info >= (3, 11): # pragma: no cover (py311+) + from typing import Self + else: # pragma: no cover ( None: # noqa: C901 + loc: dict[str, Any] = {} + glob: dict[str, Any] = {} + mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text(encoding="utf-8") + exec(mock_value_str, glob, loc) # noqa: S102 + enum_collect: dict[int, list[str | OSError]] = loc["enum_collect"] + value_collect: dict[int, dict[str, tuple[str, int] | OSError]] = loc["value_collect"] + key_open: dict[int, dict[str, int | OSError]] = loc["key_open"] + hive_open: dict[tuple[int, str, int, int], int | OSError] = loc["hive_open"] + + class Key: + def __init__(self, value: int) -> None: + self.value = value + + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + return None + + def _enum_key(key: int | Key, at: int) -> str: + key_id = key.value if isinstance(key, Key) else key + result = enum_collect[key_id][at] + if isinstance(result, OSError): + raise result + return result + + def _query_value_ex(key: Key, value_name: str) -> tuple[str, int]: + key_id = key.value if isinstance(key, Key) else key + result = value_collect[key_id][value_name] + if isinstance(result, OSError): + raise result + return result + + @contextmanager + def _open_key_ex(*args: str | int) -> Iterator[Key | int]: + if len(args) == 2: + key, value = cast(int, args[0]), cast(str, args[1]) + key_id = key.value if isinstance(key, Key) else key + got = key_open[key_id][value] + if isinstance(got, OSError): + raise got + yield Key(got) # this needs to be something that can be withed, so let's wrap it + elif len(args) == 4: # pragma: no branch + got = hive_open[cast(Tuple[int, str, int, int], args)] + if isinstance(got, OSError): + raise got + yield got + else: + raise RuntimeError # pragma: no cover + + mocker.patch("py_discovery._windows.pep514.EnumKey", side_effect=_enum_key) + mocker.patch("py_discovery._windows.pep514.QueryValueEx", side_effect=_query_value_ex) + mocker.patch("py_discovery._windows.pep514.OpenKeyEx", side_effect=_open_key_ex) + mocker.patch("os.path.exists", return_value=True) + + +@pytest.mark.usefixtures("_mock_registry") +@pytest.mark.parametrize( + ("string_spec", "expected_exe"), + [ + # 64-bit over 32-bit + ("python3.10", "C:\\Users\\user\\Miniconda3-64\\python.exe"), + ("cpython3.10", "C:\\Users\\user\\Miniconda3-64\\python.exe"), + # 1 installation of 3.9 available + ("python3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + ("cpython3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + # resolves to the highest available version + ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + # Non-standard org name + ("python3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), + ("cpython3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), + ], +) +def test_propose_interpreters(string_spec: str, expected_exe: str, mocker: MockerFixture) -> None: + mocker.patch("sys.platform", "win32") + mocker.patch("sysconfig.get_config_var", return_value=f"{sys.version_info.major}{sys.version_info}") + mocker.patch("sysconfig.get_makefile_filename", return_value="make") + + spec = PythonSpec.from_string_spec(string_spec) + mocker.patch( + "py_discovery._windows.Pep514PythonInfo.from_exe", + return_value=_mock_pyinfo(spec.major, spec.minor, spec.architecture, expected_exe), + ) + + interpreter = next(propose_interpreters(spec, env={})) + assert interpreter.executable == expected_exe + + +def _mock_pyinfo(major: int | None, minor: int | None, arch: int | None, exe: str) -> PythonInfo: + """Return PythonInfo objects with essential metadata set for the given args""" + info = PythonInfo() + info.base_prefix = str(Path(exe).parent) + info.executable = info.original_executable = info.system_executable = exe + info.implementation = "CPython" + info.architecture = arch or 64 + info.version_info = VersionInfo(major, minor, 0, "final", 0) + return info + + +@pytest.mark.usefixtures("_mock_registry") +def test_pep514() -> None: + interpreters = list(discover_pythons()) + assert interpreters == [ + ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), + ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), + ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), + ("PythonCore", 3, 8, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", None), + ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), + ("PythonCore", 3, 10, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", None), + ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None), + ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), + ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None), + ("PythonCore", 3, 7, 64, "C:\\Python37\\python.exe", None), + ] + + +@pytest.mark.usefixtures("_mock_registry") +def test_pep514_run(capsys: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture) -> None: + pep514._run() # noqa: SLF001 + out, err = capsys.readouterr() + expected = textwrap.dedent( + r""" + ('CompanyA', 3, 6, 64, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None) + ('ContinuumAnalytics', 3, 10, 32, 'C:\\Users\\user\\Miniconda3\\python.exe', None) + ('ContinuumAnalytics', 3, 10, 64, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None) + ('PythonCore', 2, 7, 64, 'C:\\Python27\\python.exe', None) + ('PythonCore', 3, 10, 32, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None) + ('PythonCore', 3, 12, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None) + ('PythonCore', 3, 7, 64, 'C:\\Python37\\python.exe', None) + ('PythonCore', 3, 8, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None) + ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + """, + ).strip() + assert out.strip() == expected + assert not err + prefix = "PEP-514 violation in Windows Registry at " + expected_logs = [ + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.1/SysArchitecture error: invalid format magic", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.2/SysArchitecture error: arch is not string: 100", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.3 error: no ExecutablePath or default for it", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.3 error: could not load exe with value None", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.11/InstallPath error: missing", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.12/SysVersion error: invalid format magic", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.X/SysVersion error: version is not string: 2778", + f"{prefix}HKEY_CURRENT_USER/PythonCore/3.X error: invalid format 3.X", + ] + assert caplog.messages == expected_logs diff --git a/tests/windows/winreg-mock-values.py b/tests/windows/winreg-mock-values.py new file mode 100644 index 0000000..cc4e81e --- /dev/null +++ b/tests/windows/winreg-mock-values.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from py_discovery._windows.pep514 import ( + HKEY_CURRENT_USER, + HKEY_LOCAL_MACHINE, + KEY_READ, + KEY_WOW64_32KEY, + KEY_WOW64_64KEY, +) + +hive_open = { + (HKEY_CURRENT_USER, "Software\\Python", 0, KEY_READ): 78701856, + (HKEY_LOCAL_MACHINE, "Software\\Python", 0, KEY_READ | KEY_WOW64_64KEY): 78701840, + (HKEY_LOCAL_MACHINE, "Software\\Python", 0, KEY_READ | KEY_WOW64_32KEY): OSError( + 2, + "The system cannot find the file specified", + ), +} +key_open = { + 78701152: { + "Anaconda310-32\\InstallPath": 78703200, + "Anaconda310-32": 78703568, + "Anaconda310-64\\InstallPath": 78703520, + "Anaconda310-64": 78702368, + }, + 78701856: {"ContinuumAnalytics": 78701152, "PythonCore": 78702656, "CompanyA": 88800000}, + 78702656: { + "3.1\\InstallPath": 78701824, + "3.1": 78700704, + "3.2\\InstallPath": 78704048, + "3.2": 78704368, + "3.3\\InstallPath": 78701936, + "3.3": 78703024, + "3.8\\InstallPath": 78703792, + "3.8": 78701792, + "3.9\\InstallPath": 78701888, + "3.9": 78703424, + "3.10-32\\InstallPath": 78703600, + "3.10-32": 78704512, + "3.11\\InstallPath": OSError(2, "The system cannot find the file specified"), + "3.11": 78700656, + "3.12\\InstallPath": 78703632, + "3.12": 78702608, + "3.X": 78703088, + }, + 78702960: {"2.7\\InstallPath": 78700912, "2.7": 78703136, "3.7\\InstallPath": 78703648, "3.7": 78704032}, + 78701840: {"PythonCore": 78702960}, + 88800000: { + "3.6\\InstallPath": 88810000, + "3.6": 88820000, + }, +} +value_collect = { + 78703568: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1)}, + 78703200: { + "ExecutablePath": ("C:\\Users\\user\\Miniconda3\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78702368: {"SysVersion": ("3.10", 1), "SysArchitecture": ("64bit", 1)}, + 78703520: { + "ExecutablePath": ("C:\\Users\\user\\Miniconda3-64\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78700704: {"SysVersion": ("3.9", 1), "SysArchitecture": ("magic", 1)}, + 78701824: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78704368: {"SysVersion": ("3.9", 1), "SysArchitecture": (100, 4)}, + 78704048: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78703024: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1)}, + 78701936: { + "ExecutablePath": OSError(2, "The system cannot find the file specified"), + "": OSError(2, "The system cannot find the file specified"), + }, + 78701792: { + "SysVersion": OSError(2, "The system cannot find the file specified"), + "SysArchitecture": OSError(2, "The system cannot find the file specified"), + }, + 78703792: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78703424: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1)}, + 78701888: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78704512: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1)}, + 78703600: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78700656: { + "SysVersion": OSError(2, "The system cannot find the file specified"), + "SysArchitecture": OSError(2, "The system cannot find the file specified"), + }, + 78702608: {"SysVersion": ("magic", 1), "SysArchitecture": ("64bit", 1)}, + 78703632: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78703088: {"SysVersion": (2778, 11)}, + 78703136: { + "SysVersion": OSError(2, "The system cannot find the file specified"), + "SysArchitecture": OSError(2, "The system cannot find the file specified"), + }, + 78700912: { + "ExecutablePath": OSError(2, "The system cannot find the file specified"), + "": ("C:\\Python27\\", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 78704032: { + "SysVersion": OSError(2, "The system cannot find the file specified"), + "SysArchitecture": OSError(2, "The system cannot find the file specified"), + }, + 78703648: { + "ExecutablePath": OSError(2, "The system cannot find the file specified"), + "": ("C:\\Python37\\", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 88810000: { + "ExecutablePath": ("Z:\\CompanyA\\Python\\3.6\\python.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, + 88820000: {"SysVersion": ("3.6", 1), "SysArchitecture": ("64bit", 1)}, +} +enum_collect = { + 78701856: [ + "ContinuumAnalytics", + "PythonCore", + "CompanyA", + OSError(22, "No more data is available", None, 259, None), + ], + 78701152: ["Anaconda310-32", "Anaconda310-64", OSError(22, "No more data is available", None, 259, None)], + 78702656: [ + "3.1", + "3.2", + "3.3", + "3.8", + "3.9", + "3.10-32", + "3.11", + "3.12", + "3.X", + OSError(22, "No more data is available", None, 259, None), + ], + 78701840: ["PyLauncher", "PythonCore", OSError(22, "No more data is available", None, 259, None)], + 78702960: ["2.7", "3.7", OSError(22, "No more data is available", None, 259, None)], + 88800000: ["3.6", OSError(22, "No more data is available", None, 259, None)], +} diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..b5829e4 --- /dev/null +++ b/tox.ini @@ -0,0 +1,86 @@ +[tox] +requires = + tox>=4.2 +env_list = + fix + py312 + py311 + py310 + py39 + py38 + py37 + type + docs + pkg_meta +skip_missing_interpreters = true + +[testenv] +description = run the tests with pytest under {envname} +package = wheel +wheel_build_env = .pkg +extras = + testing +pass_env = + FORCE_COLOR + PYTEST_* + SSL_CERT_FILE +set_env = + COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}{/}.coverage.{envname}} +commands = + pytest {tty:--color=yes} {posargs: --no-cov-on-fail --cov-context=test \ + --cov={envsitepackagesdir}{/}py_discovery --cov={toxinidir}{/}tests --cov-config={toxinidir}{/}pyproject.toml \ + --cov-report=term-missing:skip-covered --cov-report=html:{envtmpdir}{/}htmlcov \ + --cov-report=xml:{toxworkdir}{/}coverage.{envname}.xml --junitxml={toxworkdir}{/}junit.{envname}.xml \ + tests} +labels = test + +[testenv:fix] +description = run formatter and linters +skip_install = true +deps = + pre-commit>=3.6 +pass_env = + {[testenv]passenv} + PROGRAMDATA +commands = + pre-commit run --all-files --show-diff-on-failure {tty:--color=always} {posargs} + +[testenv:type] +description = run type check on code base +deps = + mypy==1.7.1 +set_env = + {tty:MYPY_FORCE_COLOR = 1} +commands = + mypy src/py_discovery + mypy tests + +[testenv:docs] +description = build documentation +extras = + docs +commands = + sphinx-build -d "{envtmpdir}{/}doctree" docs --color -b html -W {posargs:"{toxworkdir}{/}docs_out"} + python -c 'print(r"documentation available under {posargs:file://{toxworkdir}{/}docs_out}{/}index.html")' + +[testenv:pkg_meta] +description = check that the long description is valid +skip_install = true +deps = + build[virtualenv]>=1.0.3 + check-wheel-contents>=0.6 + twine>=4.0.2 +commands = + python -m build -o {envtmpdir} -s -w . + twine check --strict {envtmpdir}{/}* + check-wheel-contents --no-config {envtmpdir} + +[testenv:dev] +description = dev environment with all deps at {envdir} +package = editable +extras = + docs + testing +commands = + python -m pip list --format=columns + python -c "print(r'{envpython}')"