From 477b335285c56e39fb0cadfc0e13fe95760a3259 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 30 Oct 2022 20:56:17 +0100 Subject: [PATCH] ENH: delegate computing wheel tags to the packaging module Use the wheel contents only to determine whether the wheel contains python ABI dependent modules or other platform dependent code. Fixes #189, #190. --- meson.build | 1 - mesonpy/__init__.py | 219 +++++++++++--------------------------------- mesonpy/_compat.py | 7 +- mesonpy/_tags.py | 134 --------------------------- tests/test_tags.py | 75 --------------- tests/test_wheel.py | 35 +------ 6 files changed, 60 insertions(+), 411 deletions(-) delete mode 100644 mesonpy/_tags.py delete mode 100644 tests/test_tags.py diff --git a/meson.build b/meson.build index 27479245e..1610f29e5 100644 --- a/meson.build +++ b/meson.build @@ -11,7 +11,6 @@ py.install_sources( 'mesonpy/__init__.py', 'mesonpy/_compat.py', 'mesonpy/_elf.py', - 'mesonpy/_tags.py', 'mesonpy/_util.py', subdir: 'mesonpy', ) diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 6a271d781..3fd492f39 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -13,6 +13,7 @@ import collections import contextlib import functools +import importlib.machinery import io import itertools import json @@ -35,6 +36,8 @@ Union ) +import packaging.tags + if sys.version_info < (3, 11): import tomli as tomllib @@ -43,10 +46,9 @@ import mesonpy._compat import mesonpy._elf -import mesonpy._tags import mesonpy._util -from mesonpy._compat import Collection, Iterator, Mapping, Path +from mesonpy._compat import Iterator, Path if typing.TYPE_CHECKING: # pragma: no cover @@ -102,9 +104,8 @@ def _init_colors() -> Dict[str, str]: _STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS -_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P.+)\.(?P.+)\.so$') -_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P.+)\.(?P.+)\.pyd$') -_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P[0-9]+)$') +_EXTENSION_SUFFIXES = frozenset(s.lstrip('.') for s in importlib.machinery.EXTENSION_SUFFIXES) +_EXTENSION_SUFFIX_REGEX = re.compile(r'^(?:(?P[^.]+)\.)?(?:so|pyd)$') def _showwarning( @@ -179,6 +180,11 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]: def _has_internal_libs(self) -> bool: return bool(self._wheel_files['mesonpy-libs']) + @property + def _has_extension_modules(self) -> bool: + # Assume that all code installed in {platlib} is Python ABI dependent. + return bool(self._wheel_files['platlib']) + @property def basename(self) -> str: """Normalized wheel name and version (eg. meson_python-1.0.0).""" @@ -187,14 +193,30 @@ def basename(self) -> str: version=self._project.version, ) + @property + def tags(self) -> packaging.tags.Tag: + """Wheel tags.""" + if self.is_pure: + return packaging.tags.Tag('py3', 'none', 'any') + # Get the most specific tag for the Python interpreter. + tag = next(packaging.tags.sys_tags()) + if not self._has_extension_modules: + # The wheel has platform dependent code (is not pure) but + # does not contain any extension module (does not + # distribute any file in {platlib}) thus use generic + # implementation and ABI tags. + return packaging.tags.Tag('py3', 'none', tag.platform) + elif self._use_stable_abi: + # All distributed extension modules use the stable ABI. + return packaging.tags.Tag(tag.interpreter, 'abi3', tag.platform) + return tag + @property def name(self) -> str: """Wheel name, this includes the basename and tags.""" - return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format( + return '{basename}-{tags}'.format( basename=self.basename, - python_tag=self.python_tag, - abi_tag=self.abi_tag, - platform_tag=self.platform_tag, + tags=self.tags, ) @property @@ -229,7 +251,7 @@ def wheel(self) -> bytes: # noqa: F811 Tag: {tags} ''').strip().format( is_purelib='true' if self.is_pure else 'false', - tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}', + tags=self.tags, ).encode() @property @@ -267,166 +289,31 @@ def _debian_python(self) -> bool: except ModuleNotFoundError: return False - @property - def python_tag(self) -> str: - selected_tag = self._select_abi_tag() - if selected_tag and selected_tag.python: - return selected_tag.python - return 'py3' - - @property - def abi_tag(self) -> str: - selected_tag = self._select_abi_tag() - if selected_tag: - return selected_tag.abi - return 'none' - @cached_property - def platform_tag(self) -> str: - if self.is_pure: - return 'any' - # XXX: Choose the sysconfig platform here and let something like auditwheel - # fix it later if there are system dependencies (eg. replace it with a manylinux tag) - platform_ = sysconfig.get_platform() - parts = platform_.split('-') - if parts[0] == 'macosx': - target = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if target: - print( - '{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the ' - 'platform tag to {target}{reset}'.format(target=target, **_STYLES) - ) - parts[1] = target - else: - # If no target macOS version is specified fallback to - # platform.mac_ver() instead of sysconfig.get_platform() as the - # latter specifies the target macOS version Python was built - # against. - parts[1] = platform.mac_ver()[0] - if parts[1] >= '11': - # Only pick up the major version, which changed from 10.X - # to X.0 from macOS 11 onwards. See - # https://github.com/mesonbuild/meson-python/issues/160 - parts[1] = parts[1].split('.')[0] - - if parts[1] in ('11', '12'): - # Workaround for bug where pypa/packaging does not consider macOS - # tags without minor versions valid. Some Python flavors (Homebrew - # for example) on macOS started to do this in version 11, and - # pypa/packaging should handle things correctly from version 13 and - # forward, so we will add a 0 minor version to MacOS 11 and 12. - # https://github.com/mesonbuild/meson-python/issues/91 - # https://github.com/pypa/packaging/issues/578 - parts[1] += '.0' - - platform_ = '-'.join(parts) - elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff: - # 32-bit Python running on an x86_64 host - # https://github.com/mesonbuild/meson-python/issues/123 - parts[1] = 'i686' - platform_ = '-'.join(parts) - return platform_.replace('-', '_').replace('.', '_') - - def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the Windows tag from the Python extension file name.""" - match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename) - if not match: - return None - tag = match.group('tag') + def _use_stable_abi(self) -> bool: + """Determine whether the package is compatible with the stabe ABI. - try: - return mesonpy._tags.StableABITag(tag) - except ValueError: - return mesonpy._tags.InterpreterTag(tag) - - def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the Posix tag from the Python extension file name.""" - # sysconfig is not guaranted to export SHLIB_SUFFIX but let's be - # preventive and check its value to make sure it matches our expectations - try: - extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so') - if extension != '.so': - raise NotImplementedError( - f"We don't currently support the {extension} extension. " - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include information about your operating system.' - ) - except KeyError: - warnings.warn( - 'sysconfig does not export SHLIB_SUFFIX, so we are unable to ' - 'perform the sanity check regarding the extension suffix. ' - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include the output of `python -m sysconfig`.' - ) - match = _LINUX_NATIVE_MODULE_REGEX.match(filename) - if not match: # this file does not appear to be a native module - return None - tag = match.group('tag') + Examine all files installed in {platlib} that look like + extension modules (extension .pyd on Windows and .so on other + platforms) and return True only if all of them have a PEP 3149 + filename tag indicating that they ase the stable Python ABI. - try: - return mesonpy._tags.StableABITag(tag) - except ValueError: - return mesonpy._tags.InterpreterTag(tag) - - def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the ABI tag from the Python extension file name.""" - if os.name == 'nt': - return self._calculate_file_abi_tag_heuristic_windows(filename) - # everything else *should* follow the POSIX way, at least to my knowledge - return self._calculate_file_abi_tag_heuristic_posix(filename) - - def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str: - if len(files) > max_count: - files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))'] - return ''.join(f'{prefix}- {file}\n' for file in files) - - def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]: - """Map files into ABI tags.""" - files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list) - - for _, file in self._wheel_files['platlib']: - # if in platlib, calculate the ABI tag - tag = self._calculate_file_abi_tag_heuristic(file) - if tag: - files_by_tag[tag].append(file) - - return files_by_tag - - def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]: # noqa: C901 - """Given a list of ABI tags, selects the most specific one. - - Raises an error if there are incompatible tags. + All files that look like extension modules are verified to + have a file name compatibel with what is expected by the + Python interpreter. An exception is raised otherwise. + + Other files are ignored. """ - # Possibilities: - # - interpreter specific (cpython/pypy/etc, version) - # - stable abi (abiX) - tags = self._files_by_tag() - selected_tag = None - for tag, files in tags.items(): - # no selected tag yet, let's assign this one - if not selected_tag: - selected_tag = tag - # interpreter tag - elif isinstance(tag, mesonpy._tags.InterpreterTag): - if tag != selected_tag: - if isinstance(selected_tag, mesonpy._tags.InterpreterTag): - raise ValueError( - 'Found files with incompatible ABI tags:\n' - + self._file_list_repr(tags[selected_tag]) - + '\tand\n' - + self._file_list_repr(files) - ) - selected_tag = tag - # stable ABI - elif isinstance(tag, mesonpy._tags.StableABITag): - if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag: - raise ValueError( - 'Found files with incompatible ABI tags:\n' - + self._file_list_repr(tags[selected_tag]) - + '\tand\n' - + self._file_list_repr(files) - ) - return selected_tag + abis = [] + + for path, src in self._wheel_files['platlib']: + if sys.platform == 'nt' and path.suffix == '.pyd' or path.suffix == '.so': + name, suffix = path.name.split('.', 1) + if suffix not in _EXTENSION_SUFFIXES: + raise ValueError('Extension module "{}" not compatible with Python interpreter.'.format(str(path))) + abis.append(_EXTENSION_SUFFIX_REGEX.match(suffix).group('abi')) + + return all(abi == 'abi3' for abi in abis) def _is_native(self, file: Union[str, pathlib.Path]) -> bool: """Check if file is a native file.""" diff --git a/mesonpy/_compat.py b/mesonpy/_compat.py index 9fab9054f..dc74de40b 100644 --- a/mesonpy/_compat.py +++ b/mesonpy/_compat.py @@ -10,11 +10,9 @@ if sys.version_info >= (3, 9): - from collections.abc import ( - Collection, Iterable, Iterator, Mapping, Sequence - ) + from collections.abc import Collection, Iterable, Iterator, Sequence else: - from typing import Collection, Iterable, Iterator, Mapping, Sequence + from typing import Collection, Iterable, Iterator, Sequence if sys.version_info >= (3, 8): @@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool: 'Iterable', 'Iterator', 'Literal', - 'Mapping', 'Path', 'Sequence', ] diff --git a/mesonpy/_tags.py b/mesonpy/_tags.py deleted file mode 100644 index d9a9b3ba8..000000000 --- a/mesonpy/_tags.py +++ /dev/null @@ -1,134 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Quansight, LLC -# SPDX-FileCopyrightText: 2021 Filipe Laíns - -import abc -import re - -from typing import Any, Optional - -from mesonpy._compat import Literal, Sequence - - -class Tag(abc.ABC): - @abc.abstractmethod - def __init__(self, value: str) -> None: ... - - @abc.abstractmethod - def __str__(self) -> str: ... - - @property - @abc.abstractmethod - def python(self) -> Optional[str]: - """Python tag.""" - - @property - @abc.abstractmethod - def abi(self) -> str: - """ABI tag.""" - - -class StableABITag(Tag): - _REGEX = re.compile(r'^abi(?P[0-9]+)$') - - def __init__(self, value: str) -> None: - match = self._REGEX.match(value) - if not match: - raise ValueError(f'Invalid PEP 3149 stable ABI tag, expecting pattern `{self._REGEX.pattern}`') - self._abi_number = int(match.group('abi_number')) - - @property - def abi_number(self) -> int: - return self._abi_number - - def __str__(self) -> str: - return f'abi{self.abi_number}' - - @property - def python(self) -> Literal[None]: - return None - - @property - def abi(self) -> str: - return f'abi{self.abi_number}' - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and other.abi_number == self.abi_number - - def __hash__(self) -> int: - return hash(str(self)) - - -class InterpreterTag(Tag): - def __init__(self, value: str) -> None: - parts = value.split('-') - if len(parts) < 2: - raise ValueError( - 'Invalid PEP 3149 interpreter tag, expected at ' - f'least 2 parts but got {len(parts)}' - ) - - # On Windows, file extensions look like `.cp311-win_amd64.pyd`, so the - # implementation part (`cpython-`) is different from Linux. Handle that here: - if parts[0].startswith('cp3'): - parts.insert(0, 'cpython') - parts[1] = parts[1][2:] # strip 'cp' - - self._implementation = parts[0] - self._interpreter_version = parts[1] - self._additional_information = parts[2:] - - if self.implementation != 'cpython' and not self.implementation.startswith('pypy'): - raise NotImplementedError( - f'Unknown Python implementation: {self.implementation}. ' - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include information about the Python distribution you are using.' - ) - - @property - def implementation(self) -> str: - return self._implementation - - @property - def interpreter_version(self) -> str: - return self._interpreter_version - - @property - def additional_information(self) -> Sequence[str]: - return tuple(self._additional_information) - - def __str__(self) -> str: - return '-'.join(( - self.implementation, - self.interpreter_version, - *self.additional_information, - )) - - @property - def python(self) -> str: - if self.implementation == 'cpython': - # The Python tag for CPython does not seem to include the flags suffixes. - return f'cp{self.interpreter_version}'.rstrip('dmu') - elif self.implementation.startswith('pypy'): - return f'pp{self.implementation[4:]}' - # XXX: FYI older PyPy version use the following model - # pp{self.implementation[4]}{self.interpreter_version[2:]} - raise ValueError(f'Unknown implementation: {self.implementation}') - - @property - def abi(self) -> str: - if self.implementation == 'cpython': - return f'cp{self.interpreter_version}' - elif self.implementation.startswith('pypy'): - return f'{self.implementation}_{self.interpreter_version}' - raise ValueError(f'Unknown implementation: {self.implementation}') - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, self.__class__) - and other.implementation == self.implementation - and other.interpreter_version == self.interpreter_version - ) - - def __hash__(self) -> int: - return hash(str(self)) diff --git a/tests/test_tags.py b/tests/test_tags.py deleted file mode 100644 index 7b7fdd57a..000000000 --- a/tests/test_tags.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-License-Identifier: MIT - -import re -import sys - -import pytest - -import mesonpy._tags - - -INTERPRETER_VERSION = f'{sys.version_info[0]}{sys.version_info[1]}' - - -@pytest.mark.parametrize( - ('value', 'number', 'abi', 'python'), - [ - ('abi3', 3, 'abi3', None), - ('abi4', 4, 'abi4', None), - ] -) -def test_stable_abi_tag(value, number, abi, python): - tag = mesonpy._tags.StableABITag(value) - assert str(tag) == value - assert tag.abi_number == number - assert tag.abi == abi - assert tag.python == python - assert tag == mesonpy._tags.StableABITag(value) - - -def test_stable_abi_tag_invalid(): - with pytest.raises(ValueError, match=re.escape( - r'Invalid PEP 3149 stable ABI tag, expecting pattern `^abi(?P[0-9]+)$`' - )): - mesonpy._tags.StableABITag('invalid') - - -@pytest.mark.parametrize( - ('value', 'implementation', 'version', 'additional', 'abi', 'python'), - [ - ('cpython-37-x86_64-linux-gnu', 'cpython', '37', ('x86_64', 'linux', 'gnu'), 'cp37', 'cp37'), - ('cpython-310-x86_64-linux-gnu', 'cpython', '310', ('x86_64', 'linux', 'gnu'), 'cp310', 'cp310'), - ('cpython-310', 'cpython', '310', (), 'cp310', 'cp310'), - ('cp311-win_amd64', 'cpython', '311', ('win_amd64', ), 'cp311', 'cp311'), - ('cpython-311-win_amd64', 'cpython', '311', ('win_amd64', ), 'cp311', 'cp311'), - ('cpython-310-special', 'cpython', '310', ('special',), 'cp310', 'cp310'), - ('cpython-310-x86_64-linux-gnu', 'cpython', '310', ('x86_64', 'linux', 'gnu'), 'cp310', 'cp310'), - ('pypy39-pp73-x86_64-linux-gnu', 'pypy39', 'pp73', ('x86_64', 'linux', 'gnu'), 'pypy39_pp73', 'pp39'), - ('pypy39-pp73-win_amd64', 'pypy39', 'pp73', ('win_amd64', ), 'pypy39_pp73', 'pp39'), - ('pypy38-pp73-darwin', 'pypy38', 'pp73', ('darwin', ), 'pypy38_pp73', 'pp38'), - ] -) -def test_interpreter_tag(value, implementation, version, additional, abi, python): - tag = mesonpy._tags.InterpreterTag(value) - if not value.startswith('cp311'): - # Avoid testing the workaround for the invalid Windows tag - assert str(tag) == value - - assert tag.implementation == implementation - assert tag.interpreter_version == version - assert tag.additional_information == additional - assert tag.abi == abi - assert tag.python == python - assert tag == mesonpy._tags.InterpreterTag(value) - - -@pytest.mark.parametrize( - ('value', 'msg'), - [ - ('', 'Invalid PEP 3149 interpreter tag, expected at least 2 parts but got 1'), - ('invalid', 'Invalid PEP 3149 interpreter tag, expected at least 2 parts but got 1'), - ] -) -def test_interpreter_tag_invalid(value, msg): - with pytest.raises(ValueError, match=msg): - mesonpy._tags.InterpreterTag(value) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 1fed555f5..22a0390e9 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -8,6 +8,7 @@ import sysconfig import textwrap +import packaging.tags import pytest import wheel.wheelfile @@ -25,35 +26,10 @@ EXT_SUFFIX = sysconfig.get_config_var('EXT_SUFFIX') INTERPRETER_VERSION = f'{sys.version_info[0]}{sys.version_info[1]}' - -if platform.python_implementation() == 'CPython': - INTERPRETER_TAG = f'cp{INTERPRETER_VERSION}' - PYTHON_TAG = INTERPRETER_TAG - # Py_UNICODE_SIZE has been a runtime option since Python 3.3, - # so the u suffix no longer exists - if sysconfig.get_config_var('Py_DEBUG'): - INTERPRETER_TAG += 'd' - # https://github.com/pypa/packaging/blob/5984e3b25f4fdee64aad20e98668c402f7ed5041/packaging/tags.py#L147-L150 - if sys.version_info < (3, 8): - pymalloc = sysconfig.get_config_var('WITH_PYMALLOC') - if pymalloc or pymalloc is None: # none is the default value, which is enable - INTERPRETER_TAG += 'm' -elif platform.python_implementation() == 'PyPy': - INTERPRETER_TAG = sysconfig.get_config_var('SOABI').replace('-', '_') - PYTHON_TAG = f'pp{INTERPRETER_VERSION}' -else: - raise NotImplementedError(f'Unknown implementation: {platform.python_implementation()}') - -platform_ = sysconfig.get_platform() -if platform.system() == 'Darwin': - parts = platform_.split('-') - parts[1] = platform.mac_ver()[0] - if parts[1] >= '11': - parts[1] = parts[1].split('.')[0] - if parts[1] in ('11', '12'): - parts[1] += '.0' - platform_ = '-'.join(parts) -PLATFORM_TAG = platform_.replace('-', '_').replace('.', '_') +_tag = next(packaging.tags.sys_tags()) +INTERPRETER_TAG = _tag.abi +PYTHON_TAG = _tag.interpreter +PLATFORM_TAG = _tag.platform if platform.system() == 'Linux': SHARED_LIB_EXT = 'so' @@ -80,7 +56,6 @@ def wheel_filename(artifact): win_py37 = os.name == 'nt' and sys.version_info < (3, 8) -@pytest.mark.skipif(win_py37, reason='An issue with missing file extension') def test_scipy_like(wheel_scipy_like): # This test is meant to exercise features commonly needed by a regular # Python package for scientific computing or data science: