Fix release linux ci #19
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and deploy python wheels to PyPI | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| push: | |
| tags: | |
| - "v*.*.*" | |
| jobs: | |
| build_manylinux_wheels: | |
| name: Build linux wheels with ${{ matrix.py_ver_prefix }}-${{ matrix.linux_arch }}-${{ matrix.device }} | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| py_ver_prefix: [cp36, cp37, cp38, cp39, cp310] | |
| linux_arch: [x86_64, i686, aarch64, s390x, ppc64le] #the maximum timeout of a job is 360 minutes, which is easy to time out, so we compile different system architectures separately. | |
| device: [cpu, gpu] | |
| exclude: | |
| - linux_arch: i686 | |
| device: gpu | |
| - linux_arch: aarch64 | |
| device: gpu | |
| - linux_arch: s390x | |
| device: gpu | |
| - linux_arch: ppc64le | |
| device: gpu | |
| fail-fast: false | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Check out source code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Get python path | |
| run: | | |
| sudo apt update | |
| sudo apt install -y jq | |
| echo PY_PATH=$(echo '{"cp36": "cp36-cp36m", "cp37": "cp37-cp37m", "cp38": "cp38-cp38", "cp39": "cp39-cp39", "cp310": "cp310-cp310", "cp311": "cp311-cp311", "cp312": "cp312-cp312", "pp37": "pp37-pypy37_pp73", "pp38": "pp38-pypy38_pp73", "pp39": "pp39-pypy39_pp73"}' | jq .${{matrix.py_ver_prefix}} ) >> $GITHUB_ENV | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Get necessary envs that the current process has access to | |
| id: set_env | |
| run: | | |
| from os import environ, sched_getaffinity | |
| # get the cpu cores | |
| num_cpus = len(sched_getaffinity(0)) | |
| output_file = environ["GITHUB_OUTPUT"] | |
| with open(output_file, "a", encoding="utf-8") as output_stream: | |
| output_stream.write(f"count={num_cpus}\n") | |
| shell: python | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| env: | |
| CIBW_ENVIRONMENT: > | |
| LD_LIBRARY_PATH=/opt/python/${{ env.PY_PATH }}/lib:/opt/python/cp38-cp38/lib:/opt/python/cp310-cp310/lib | |
| CMAKE_BUILD_PARALLEL_LEVEL=${{ steps.set_env.outputs.count }} | |
| GIT_SHA=${{ github.sha }} | |
| DEVICE=${{ matrix.device }} | |
| CUDA_PATH=/usr/local/cuda | |
| SCRIPT_EXEC_MODE=x86 | |
| CIBW_BUILD_VERBOSITY: 1 | |
| CIBW_BUILD: ${{ matrix.py_ver_prefix }}-* | |
| CIBW_SKIP: "*musllinux*" #disable musllinux now | |
| CIBW_ARCHS_LINUX: ${{ matrix.linux_arch }} | |
| #CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_x86_64_shared_python:latest | |
| #CIBW_MANYLINUX_I686_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_i686_shared_python:latest | |
| #CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_aarch64_shared_python:latest | |
| #CIBW_MANYLINUX_PPC64LE_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_ppc64le_shared_python:latest | |
| #CIBW_MANYLINUX_S390X_IMAGE: ghcr.io/${{ github.repository }}_manylinux2014_s390x_shared_python:latest | |
| CIBW_MANYLINUX_X86_64_IMAGE: smartbrave/manylinux2014_x86_64_shared_python:latest | |
| CIBW_MANYLINUX_I686_IMAGE: smartbrave/manylinux2014_i686_shared_python:latest | |
| CIBW_MANYLINUX_AARCH64_IMAGE: smartbrave/manylinux2014_aarch64_shared_python:latest | |
| CIBW_MANYLINUX_PPC64LE_IMAGE: smartbrave/manylinux2014_ppc64le_shared_python:latest | |
| CIBW_MANYLINUX_S390X_IMAGE: smartbrave/manylinux2014_s390x_shared_python:latest | |
| CIBW_BEFORE_ALL_LINUX: > | |
| (./scripts/build_ffmpeg.sh --device=${{ matrix.device }}) | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: | |
| tmp_dir=$(mktemp -d) && | |
| unzip -d ${tmp_dir} {wheel} && | |
| auditwheel -v repair $(ldd ${tmp_dir}/bmf/lib/* | awk '{if(NF>1){print $1}}' | grep "avdevice\|avfilter\|avformat\|avcodec\|postproc\|swresample\|swscale\|avutil\|cuda\|libnpp\|libxcb\|libXau\|libbz2\|bmf_module_sdk\|engine\|hmp" | sort | uniq | awk '{printf(" --exclude %s", $1);}') -w {dest_dir} {wheel} && | |
| rm -rf ${tmp_dir} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| path: ./wheelhouse/*.whl | |
| build_macos_wheels: | |
| name: Build macos wheels with ${{ matrix.py_ver_prefix }}-${{ matrix.macos_arch }} | |
| runs-on: macos-12 # On the macOS-12 system, the wheel package generated by using the xcode14.0~14.3 version compiler will have problems at python runtime which installed by homebrew, and the error is reported: "Symbol not found: _PyBaseObject_Type", please refer to issue: https://github.com/pybind/pybind11/pull/4301 and https://github.com/python/cpython/issues/97524. So we use macos-11 here. | |
| strategy: | |
| matrix: | |
| py_ver_prefix: [cp36, cp37, cp38, cp39, cp310] | |
| macos_arch: [x86_64, arm64, universal2] | |
| exclude: | |
| - py_ver_prefix: cp36 | |
| macos_arch: arm64 | |
| - py_ver_prefix: cp36 | |
| macos_arch: universal2 | |
| - py_ver_prefix: cp37 | |
| macos_arch: arm64 | |
| - py_ver_prefix: cp37 | |
| macos_arch: universal2 | |
| - py_ver_prefix: cp38 # the universal python package is experimental on Intel host, so we disable cross-compiling for python3.8. | |
| macos_arch: arm64 | |
| - py_ver_prefix: cp38 | |
| macos_arch: universal2 | |
| fail-fast: false | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Check out source code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| env: | |
| #TODO get real cpu counts | |
| # HAVE_STD_REGEX and RUN_HAVE_STD_REGES is needed by google benchmark when compiling ARM64 on x86_64 for macos. | |
| CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=10 GIT_SHA=${{ github.sha }} PKG_CONFIG_PATH="$(pwd)/ffmpeg_${{ matrix.macos_arch }}/lib/pkgconfig" CMAKE_ARGS="-DHAVE_STD_REGEX=ON -DRUN_HAVE_STD_REGEX=1" | |
| CIBW_BUILD: ${{ matrix.py_ver_prefix }}-* | |
| CIBW_BUILD_VERBOSITY: 1 | |
| CIBW_ARCHS_MACOS: ${{ matrix.macos_arch }} | |
| CIBW_BEFORE_ALL_MACOS: > | |
| sed -i '' '/sigma_gn /s/^/\/\//g' bmf/hml/third_party/benchmark/src/complexity.cc && | |
| brew list python@3.$(echo ${{ matrix.py_ver_prefix }} | cut -c 4-); if [ $? -eq 0 ]; then brew uninstall --ignore-dependencies python@3.$(echo ${{ matrix.py_ver_prefix }} | cut -c 4-); fi | |
| CIBW_BEFORE_BUILD: > | |
| ./scripts/build_ffmpeg.sh --arch=${{ matrix.macos_arch }} && | |
| for xcode_python_root in /Applications/Xcode*/Contents/Developer/Library/Frameworks/Python3.framework;do if [ -d ${xcode_python_root} ]; then mv ${xcode_python_root} ${xcode_python_root}.bak; fi; done | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| path: ./wheelhouse/*.whl | |
| - name: Setup tmate session | |
| if: ${{ failure() }} | |
| uses: mxschmitt/action-tmate@v3 | |
| upload_all_to_pypi: | |
| needs: [build_manylinux_wheels, build_macos_wheels] | |
| runs-on: ubuntu-latest # pypa/gh-action-pypi-publish only support linux | |
| if: startsWith(github.ref, 'refs/tags/') | |
| #if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: artifact | |
| path: dist | |
| - name: Publish package distributions to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| print-hash: true | |
| verbose: true | |
| skip-existing: true | |
| repository-url: https://test.pypi.org/legacy/ | |
| user: __token__ | |
| password: ${{ secrets.TEST_PYPI_TOKEN }} | |
| #packages-dir: wheelhouse/ | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| print-hash: true | |
| verbose: true | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| #packages-dir: wheelhouse/ |