Skip to content

Commit

Permalink
ENH: determine wheel tags by Python interpreter introspection
Browse files Browse the repository at this point in the history 
The extension modules filename suffixes do not contain enough
information to correctly determine the wheel tags. Instead introspect
the Python interpreter to derive the wheel tags. This is the same
approach used by other PEP517 backends, most notably wheel. The wheel
contents only to determine whether the wheel contains python ABI
dependent modules or other platform dependent code.

The packaging module is the reference wheel tags derivation
implementation and it is used (or vendored) by most python packages
dealing with wheels. However, the API provided by packaging is
cumbersome to use for our purposes and, with the goal of merging this
code into Meson in the future, it is good to avoid an additional
dependency. Therefore, the tags derivation code is reimplemented.

Tests are added to verify that the tags produced by meson-python agree
with the ones produced by packaging to ensure that the two
implementations will not diverge.

Fixes mesonbuild#142, fixes mesonbuild#189, fixes mesonbuild#190.
  • Loading branch information
@dnicolodi
dnicolodi committed Nov 16, 2022
1 parent 08be6a3 commit a604d11
Show file tree
Hide file tree
Showing 6 changed files with 324 additions and 403 deletions.
223 changes: 57 additions & 166 deletions mesonpy/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import collections
import contextlib
import functools
import importlib.machinery
import io
import itertools
import json
Expand Down Expand Up @@ -46,7 +47,7 @@
import mesonpy._tags
import mesonpy._util

from mesonpy._compat import Collection, Iterator, Mapping, Path
from mesonpy._compat import Iterator, Path


if typing.TYPE_CHECKING: # pragma: no cover
Expand Down Expand Up @@ -102,9 +103,9 @@ def _init_colors() -> Dict[str, str]:
_STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS


_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.so$')
_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.pyd$')
_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P<abi_number>[0-9]+)$')
_EXTENSION_SUFFIXES = frozenset(importlib.machinery.EXTENSION_SUFFIXES)
_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P<abi>[^.]+)\.)?(?:so|pyd)$')
assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES)


def _showwarning(
Expand Down Expand Up @@ -179,6 +180,11 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]:
def _has_internal_libs(self) -> bool:
return bool(self._wheel_files['mesonpy-libs'])

@property
def _has_extension_modules(self) -> bool:
# Assume that all code installed in {platlib} is Python ABI dependent.
return bool(self._wheel_files['platlib'])

@property
def basename(self) -> str:
"""Normalized wheel name and version (eg. meson_python-1.0.0)."""
Expand All @@ -187,14 +193,25 @@ def basename(self) -> str:
version=self._project.version,
)

@property
def tag(self) -> mesonpy._tags.Tag:
"""Wheel tags."""
if self.is_pure:
return mesonpy._tags.Tag('py3', 'none', 'any')
if not self._has_extension_modules:
# The wheel has platform dependent code (is not pure) but
# does not contain any extension module (does not
# distribute any file in {platlib}) thus use generic
# implementation and ABI tags.
return mesonpy._tags.Tag('py3', 'none', None)
return mesonpy._tags.Tag(None, self._stable_abi, None)

@property
def name(self) -> str:
"""Wheel name, this includes the basename and tags."""
return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format(
"""Wheel name, this includes the basename and tag."""
return '{basename}-{tag}'.format(
basename=self.basename,
python_tag=self.python_tag,
abi_tag=self.abi_tag,
platform_tag=self.platform_tag,
tag=self.tag,
)

@property
Expand Down Expand Up @@ -226,10 +243,10 @@ def wheel(self) -> bytes: # noqa: F811
Wheel-Version: 1.0
Generator: meson
Root-Is-Purelib: {is_purelib}
Tag: {tags}
Tag: {tag}
''').strip().format(
is_purelib='true' if self.is_pure else 'false',
tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}',
tag=self.tag,
).encode()

@property
Expand Down Expand Up @@ -267,166 +284,40 @@ def _debian_python(self) -> bool:
except ModuleNotFoundError:
return False

@property
def python_tag(self) -> str:
selected_tag = self._select_abi_tag()
if selected_tag and selected_tag.python:
return selected_tag.python
return 'py3'

@property
def abi_tag(self) -> str:
selected_tag = self._select_abi_tag()
if selected_tag:
return selected_tag.abi
return 'none'

@cached_property
def platform_tag(self) -> str:
if self.is_pure:
return 'any'
# XXX: Choose the sysconfig platform here and let something like auditwheel
# fix it later if there are system dependencies (eg. replace it with a manylinux tag)
platform_ = sysconfig.get_platform()
parts = platform_.split('-')
if parts[0] == 'macosx':
target = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
if target:
print(
'{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the '
'platform tag to {target}{reset}'.format(target=target, **_STYLES)
)
parts[1] = target
else:
# If no target macOS version is specified fallback to
# platform.mac_ver() instead of sysconfig.get_platform() as the
# latter specifies the target macOS version Python was built
# against.
parts[1] = platform.mac_ver()[0]
if parts[1] >= '11':
# Only pick up the major version, which changed from 10.X
# to X.0 from macOS 11 onwards. See
# https://github.com/mesonbuild/meson-python/issues/160
parts[1] = parts[1].split('.')[0]

if parts[1] in ('11', '12'):
# Workaround for bug where pypa/packaging does not consider macOS
# tags without minor versions valid. Some Python flavors (Homebrew
# for example) on macOS started to do this in version 11, and
# pypa/packaging should handle things correctly from version 13 and
# forward, so we will add a 0 minor version to MacOS 11 and 12.
# https://github.com/mesonbuild/meson-python/issues/91
# https://github.com/pypa/packaging/issues/578
parts[1] += '.0'

platform_ = '-'.join(parts)
elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff:
# 32-bit Python running on an x86_64 host
# https://github.com/mesonbuild/meson-python/issues/123
parts[1] = 'i686'
platform_ = '-'.join(parts)
return platform_.replace('-', '_').replace('.', '_')

def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]:
"""Try to calculate the Windows tag from the Python extension file name."""
match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename)
if not match:
return None
tag = match.group('tag')
def _stable_abi(self) -> Optional[str]:
"""Determine stabe ABI compatibility.
try:
return mesonpy._tags.StableABITag(tag)
except ValueError:
return mesonpy._tags.InterpreterTag(tag)

def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]:
"""Try to calculate the Posix tag from the Python extension file name."""
# sysconfig is not guaranted to export SHLIB_SUFFIX but let's be
# preventive and check its value to make sure it matches our expectations
try:
extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so')
if extension != '.so':
raise NotImplementedError(
f"We don't currently support the {extension} extension. "
'Please report this to https://github.com/mesonbuild/mesonpy/issues '
'and include information about your operating system.'
)
except KeyError:
warnings.warn(
'sysconfig does not export SHLIB_SUFFIX, so we are unable to '
'perform the sanity check regarding the extension suffix. '
'Please report this to https://github.com/mesonbuild/mesonpy/issues '
'and include the output of `python -m sysconfig`.'
)
match = _LINUX_NATIVE_MODULE_REGEX.match(filename)
if not match: # this file does not appear to be a native module
return None
tag = match.group('tag')
Examine all files installed in {platlib} that look like
extension modules (extension .pyd on Windows and .so on other
platforms) and, if they all share the same PEP 3149 filename
stable ABI tag, return it.
try:
return mesonpy._tags.StableABITag(tag)
except ValueError:
return mesonpy._tags.InterpreterTag(tag)

def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]:
"""Try to calculate the ABI tag from the Python extension file name."""
if os.name == 'nt':
return self._calculate_file_abi_tag_heuristic_windows(filename)
# everything else *should* follow the POSIX way, at least to my knowledge
return self._calculate_file_abi_tag_heuristic_posix(filename)

def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str:
if len(files) > max_count:
files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))']
return ''.join(f'{prefix}- {file}\n' for file in files)

def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]:
"""Map files into ABI tags."""
files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list)

for _, file in self._wheel_files['platlib']:
# if in platlib, calculate the ABI tag
tag = self._calculate_file_abi_tag_heuristic(file)
if tag:
files_by_tag[tag].append(file)

return files_by_tag

def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]: # noqa: C901
"""Given a list of ABI tags, selects the most specific one.
Raises an error if there are incompatible tags.
All files that look like extension modules are verified to
have a file name compatibel with what is expected by the
Python interpreter. An exception is raised otherwise.
Other files are ignored.
"""
# Possibilities:
# - interpreter specific (cpython/pypy/etc, version)
# - stable abi (abiX)
tags = self._files_by_tag()
selected_tag = None
for tag, files in tags.items():
# no selected tag yet, let's assign this one
if not selected_tag:
selected_tag = tag
# interpreter tag
elif isinstance(tag, mesonpy._tags.InterpreterTag):
if tag != selected_tag:
if isinstance(selected_tag, mesonpy._tags.InterpreterTag):
raise ValueError(
'Found files with incompatible ABI tags:\n'
+ self._file_list_repr(tags[selected_tag])
+ '\tand\n'
+ self._file_list_repr(files)
)
selected_tag = tag
# stable ABI
elif isinstance(tag, mesonpy._tags.StableABITag):
if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag:
abis = []

for path, src in self._wheel_files['platlib']:
if os.name == 'nt' and path.suffix == '.pyd' or path.suffix == '.so':
match = re.match(r'^[^.]+(.*)$', path.name)
assert match is not None
suffix = match.group(1)
if suffix not in _EXTENSION_SUFFIXES:
raise ValueError(
'Found files with incompatible ABI tags:\n'
+ self._file_list_repr(tags[selected_tag])
+ '\tand\n'
+ self._file_list_repr(files)
)
return selected_tag
f'Extension module {str(path)!r} not compatible with Python interpreter. '
f'Filename suffix {suffix!r} not in {set(_EXTENSION_SUFFIXES)}.')
match = _EXTENSION_SUFFIX_REGEX.match(suffix)
assert match is not None
abis.append(match.group('abi'))

stable = [x for x in abis if x and re.match(r'abi\d+', x)]
if len(stable) > 0 and len(stable) == len(abis) and all(x == stable[0] for x in stable[1:]):
return stable[0]
return None

def _is_native(self, file: Union[str, pathlib.Path]) -> bool:
"""Check if file is a native file."""
Expand Down
7 changes: 2 additions & 5 deletions mesonpy/_compat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,9 @@


if sys.version_info >= (3, 9):
from collections.abc import (
Collection, Iterable, Iterator, Mapping, Sequence
)
from collections.abc import Collection, Iterable, Iterator, Sequence
else:
from typing import Collection, Iterable, Iterator, Mapping, Sequence
from typing import Collection, Iterable, Iterator, Sequence


if sys.version_info >= (3, 8):
Expand All @@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool:
'Iterable',
'Iterator',
'Literal',
'Mapping',
'Path',
'Sequence',
]
Loading

0 comments on commit a604d11

Please sign in to comment.