From 5e6807bab9ba6a8112b8ba1f92285924e699ee84 Mon Sep 17 00:00:00 2001 From: Rot127 <45763064+Rot127@users.noreply.github.com> Date: Thu, 15 Aug 2024 12:57:23 +0000 Subject: [PATCH] [next] Updates and fixes to the Python wheel builder workflow (#2441) --- .github/workflows/python-publish-release.yml | 27 ++++--- CMakeLists.txt | 17 +++++ bindings/python/setup.py | 2 +- suite/check_wheel_bin_arch.py | 79 ++++++++++++++++++++ 4 files changed, 112 insertions(+), 13 deletions(-) create mode 100755 suite/check_wheel_bin_arch.py diff --git a/.github/workflows/python-publish-release.yml b/.github/workflows/python-publish-release.yml index 0007bd5b2d..ba3db4e3fe 100644 --- a/.github/workflows/python-publish-release.yml +++ b/.github/workflows/python-publish-release.yml @@ -18,16 +18,16 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" - CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" # ppc64le s390x really slow - CIBW_ARCHS_WINDOWS: "AMD64 x86" # ARM64 Seems ARM64 will rebuild amd64 wheel for unknow reason. + CIBW_ARCHS_LINUX: "x86_64 i686" # ppc64le s390x really slow + CIBW_ARCHS_WINDOWS: "AMD64" # ARM64 Seems ARM64 will rebuild amd64 wheel for unknow reason. CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" CIBW_SKIP: "" with: @@ -36,6 +36,17 @@ jobs: - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl + name: artifacts-${{ matrix.os }} + + - name: Check binaries (Windows) + if: matrix.os == 'windows-latest' + run: | + python3.exe suite/check_wheel_bin_arch.py ./wheelhouse/ + + - name: Check binaries (Unix) + if: matrix.os != 'windows-latest' + run: | + ./suite/check_wheel_bin_arch.py ./wheelhouse/ make_sdist: name: Make SDist @@ -64,16 +75,8 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: artifact path: dist - # - name: Publish distribution 📦 to test PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # user: __token__ - # password: ${{ secrets.test_pypi_pass }} - # repository_url: https://test.pypi.org/legacy/ - - name: Publish distribution 📦 to PyPI if: ${{ success() }} uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 54d93fcd82..578aa603a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,23 @@ endif() set(SUPPORTED_ARCHITECTURES ARM AARCH64 M68K MIPS PPC SPARC SYSZ XCORE X86 TMS320C64X M680X EVM MOS65XX WASM BPF RISCV SH TRICORE ALPHA HPPA LOONGARCH) set(SUPPORTED_ARCHITECTURE_LABELS ARM AARCH64 M68K MIPS PowerPC Sparc SystemZ XCore x86 TMS320C64x M680x EVM MOS65XX WASM BPF RISCV SH TriCore Alpha HPPA LoongArch) +# If building for OSX it's best to allow CMake to handle building both architectures +if(APPLE AND NOT CAPSTONE_BUILD_MACOS_THIN) + # The cibuildwheel on Github Actions sets this env variable + # with the architecture flags it wants to build for. + if(DEFINED ENV{ARCHFLAGS}) + if("$ENV{ARCHFLAGS}" STREQUAL "-arch arm64 -arch x86_64") + set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64") + elseif("$ENV{ARCHFLAGS}" STREQUAL "-arch arm64") + set(CMAKE_OSX_ARCHITECTURES "arm64") + elseif("$ENV{ARCHFLAGS}" STREQUAL "-arch x86_64") + set(CMAKE_OSX_ARCHITECTURES "x86_64") + endif() + else() + set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64") + endif() +endif() + list(LENGTH SUPPORTED_ARCHITECTURES count) math(EXPR count "${count}-1") # create options controlling whether support for a particular architecture is needed diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 385d6dc512..764cb447b4 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -134,7 +134,7 @@ def build_libraries(): os.chdir(BUILD_DIR) - # platform description refers at https://docs.python.org/2/library/sys.html#sys.platform + # platform description refers at https://docs.python.org/3/library/sys.html#sys.platform # Use cmake for both Darwin and Windows since it can generate fat binaries if SYSTEM == "win32" or SYSTEM == 'darwin': # Windows build: this process requires few things: diff --git a/suite/check_wheel_bin_arch.py b/suite/check_wheel_bin_arch.py new file mode 100755 index 0000000000..a805d84ad2 --- /dev/null +++ b/suite/check_wheel_bin_arch.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright © 2024 Rot127 +# SPDX-License-Identifier: BSD-3 + +import logging as log +import subprocess as sp +import re +import os +import sys +from pathlib import Path + +if len(sys.argv) != 2: + print(f"{sys.argv[0]} ") + exit(-1) + +log.basicConfig( + level=log.INFO, + stream=sys.stdout, + format="%(levelname)-5s - %(message)s", + force=True, +) + +archs = { + "universal2": ["x86_64", "arm64"], + "x86_64": [r"x86[_-]64"], + "arm64": ["arm64"], + "aarch64": ["ARM aarch64"], + "i686": ["Intel 80386"], + "win32": [r"INVALID"], + "amd64": [r"x86[_-]64"], +} + +filename = { + "macosx": "libcapstone.dylib", + "manylinux": "libcapstone.so", + "win": "capstone.dll", +} + +success = True +wheel_seen = False +for root, dir, files in os.walk(sys.argv[1]): + for file in files: + f = Path(root).joinpath(file) + if f.suffix != ".whl": + continue + wheel_seen = True + target = re.search(r"py3-none-(.+).whl", f"{f}").group(1) + platform = re.search("^(win|manylinux|macosx)", target).group(1) + + arch = re.search( + "(universal2|x86_64|arm64|aarch64|i686|win32|amd64)$", target + ).group(1) + log.info(f"Target: {target} - Platform: {platform} - Arch: {archs[arch]}") + + out_dir = f"{platform}__{arch}" + sp.run(["unzip", "-q", f"{f}", "-d", out_dir], check=True) + lib_path = Path(out_dir).joinpath(f"capstone/lib/{filename[platform]}") + result = sp.run(["file", "-b", f"{lib_path}"], capture_output=True, check=True) + stdout = result.stdout.decode("utf8").strip() + if any([not re.search(a, stdout) for a in archs[arch]]): + success = False + log.error(f"The wheel '{file}' is not compiled for '{archs[arch]}'.") + log.error(f"Binary is: {stdout}") + print() + else: + log.info(f"OK: Arch: {arch} - {lib_path}") + log.info(f"Binary is: {stdout}") + log.info(f"Delete {out_dir}") + print() + os.system(f"rm -r {out_dir}") + break + +if not wheel_seen: + log.error("No wheel was checked.") + exit(-1) + +if not success: + log.error("Binary files are compiled for the wrong architecture.") + exit(-1)