ci.yaml

---
# This is the CI workflow (not the artifact build/release workflow). The workflows
# are split because GHA doesn't support a dynamic/conditional matrix. This workflow
# has slow jobs and jobs that require private GHA runners (eg, M1 Mac) commented out.defaults:
# Ensure changes are synced with manual_artifact_build.yaml.
name: PyYAML CI

on:
  push:
  pull_request:
    types: [opened, synchronize, edited, reopened]
  workflow_dispatch:

env:
  LIBYAML_REPO: https://github.com/yaml/libyaml
  LIBYAML_REF: 0.2.5

jobs:
  python_sdist:
    name: pyyaml sdist
    runs-on: ubuntu-latest
    steps:
    - name: Checkout PyYAML
      uses: actions/checkout@v2

    - name: Install a python
      uses: actions/setup-python@v2
      with:
        python-version: 3.x

    - name: Build sdist
      env:
        PYYAML_FORCE_CYTHON: 1
        PYYAML_FORCE_LIBYAML: 0
      run: |
        python -V
        python -m pip install build

        python -m build .

        # Ensure exactly one artifact was produced.
        [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || {
          echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'."
          exit 1
        }

    - name: Test sdist
      run: |
        # Install some libyaml headers.
        # TODO Should we smoke test the sdist against the libyaml we built?
        sudo apt update
        sudo apt install libyaml-dev -y

        # Ensure Cython is not present so we use only what's in the sdist.
        python -m pip uninstall Cython -y || true

        # Pass no extra args.
        # We should auto-install with libyaml since it's present.
        python -m pip install dist/*.tar.gz -v

        python packaging/build/smoketest.py

    - name: Upload sdist artifact
      uses: actions/upload-artifact@v2
      with:
        name: dist
        path: dist/*.tar.gz


  linux_libyaml:
    name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        cfg:
        - { platform: manylinux1, arch: x86_64 }
        - { platform: manylinux2014, arch: x86_64 }
#        - { platform: manylinux2014, arch: aarch64 }
#        - { platform: manylinux2014, arch: s390x }
    env:
      DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}}
    steps:
    - name: Check cached libyaml state
      id: cached_libyaml
      uses: actions/cache@v2
      with:
        path: libyaml
        key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}}

    - name: configure docker foreign arch support
      uses: docker/setup-qemu-action@v1
      if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: Checkout pyyaml
      uses: actions/checkout@v2
      if: steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: Build libyaml
      run: >
        docker run --rm
        --volume "$(pwd):/io"
        --env LIBYAML_REF
        --env LIBYAML_REPO
        --workdir /io
        "$DOCKER_IMAGE"
        /io/packaging/build/libyaml.sh
      if: steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: ensure output is world readable (or cache fill fails with Permission Denied)
      run: >
        sudo chmod -R a+r ./libyaml/
      if: steps.cached_libyaml.outputs.cache-hit != 'true'


  linux_pyyaml:
    needs: linux_libyaml
    name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
        - { platform: manylinux1, arch: x86_64, spec: cp36 }
#        - { platform: manylinux1, arch: x86_64, spec: cp37 }
#        - { platform: manylinux1, arch: x86_64, spec: cp38 }
#        - { platform: manylinux1, arch: x86_64, spec: cp39 }
#        - { platform: manylinux2014, arch: x86_64, spec: cp310 }
        - { platform: manylinux2014, arch: x86_64, spec: cp311 }
#        - { platform: manylinux2014, arch: aarch64, spec: cp36 }
#        - { platform: manylinux2014, arch: aarch64, spec: cp37 }
#        - { platform: manylinux2014, arch: aarch64, spec: cp38 }
#        - { platform: manylinux2014, arch: aarch64, spec: cp39 }
#        - { platform: manylinux2014, arch: aarch64, spec: cp310 }
#        - { platform: manylinux2014, arch: aarch64, spec: cp311 }
#        - { platform: manylinux2014, arch: s390x, spec: cp36 }
#        - { platform: manylinux2014, arch: s390x, spec: cp37 }
#        - { platform: manylinux2014, arch: s390x, spec: cp38 }
#        - { platform: manylinux2014, arch: s390x, spec: cp39 }
#        - { platform: manylinux2014, arch: s390x, spec: cp310 }
#        - { platform: manylinux2014, arch: s390x, spec: cp311 }

    steps:
    - name: Checkout PyYAML
      uses: actions/checkout@v2

    - name: Fetch cached libyaml
      id: cached_libyaml
      uses: actions/cache@v2
      with:
        path: libyaml
        key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}

    - name: Ensure libyaml fetched
      run: exit 1
      if: steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: configure docker foreign arch support
      uses: docker/setup-qemu-action@v1
      if: matrix.arch != 'x86_64'

    - name: Build/Test/Package
      env:
        CIBW_ARCHS: all
        CIBW_BUILD: ${{matrix.spec}}-manylinux_${{matrix.arch}}
        CIBW_BUILD_VERBOSITY: 1
        # containerized Linux builds require explicit CIBW_ENVIRONMENT
        CIBW_ENVIRONMENT: >
          C_INCLUDE_PATH=libyaml/include
          LIBRARY_PATH=libyaml/src/.libs
          LD_LIBRARY_PATH=libyaml/src/.libs
          PYYAML_FORCE_CYTHON=1
          PYYAML_FORCE_LIBYAML=1
        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
      run: |
        set -eux
        python3 -V
        python3 -m pip install -U --user cibuildwheel
        python3 -m cibuildwheel --platform auto --output-dir dist .

    - name: Upload artifacts
      uses: actions/upload-artifact@v2
      with:
        name: dist
        path: dist/*.whl
        if-no-files-found: error

  macos_libyaml:
    name: libyaml macos ${{matrix.arch}}
    strategy:
      matrix:
        include:
        - arch: x86_64
#        - arch: arm64
#          runs_on: [self-hosted, macOS, arm64]
#          deployment_target: '11.0'
#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#          sdkroot: macosx11.3
    defaults:
      run:
        shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
    runs-on: ${{ matrix.runs_on || 'macos-10.15' }}
    steps:
    - name: Check cached libyaml state
      id: cached_libyaml
      uses: actions/cache@v2
      with:
        path: libyaml
        key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}}

    - name: Checkout PyYAML
      uses: actions/checkout@v2
      if: steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: Build libyaml
      env:
        MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
        SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
      run: |
        set -eux
        brew install automake coreutils m4
        bash ./packaging/build/libyaml.sh
        echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)"
      if: steps.cached_libyaml.outputs.cache-hit != 'true'


  macos_pyyaml:
    needs: macos_libyaml
    name: pyyaml ${{ matrix.spec }}
    runs-on: ${{ matrix.runs_on || 'macos-10.15' }}
    defaults:
      run:
        shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
    strategy:
      matrix:
        include:
        - spec: cp36-macosx_x86_64
#        - spec: cp37-macosx_x86_64
#        - spec: cp38-macosx_x86_64
#        - spec: cp39-macosx_x86_64
#        - spec: cp310-macosx_x86_64
        - spec: cp311-macosx_x86_64

#        # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
#        # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
#        - spec: cp39-macosx_arm64
#          deployment_target: '11.0'
#          runs_on: [self-hosted, macOS, arm64]
#          arch: arm64
#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#          sdkroot: macosx11.3
#
#        - spec: cp310-macosx_arm64
#          deployment_target: '11.0'
#          runs_on: [self-hosted, macOS, arm64]
#          arch: arm64
#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#          sdkroot: macosx11.3
#
#        - spec: cp311-macosx_arm64
#          deployment_target: '11.0'
#          runs_on: [self-hosted, macOS, arm64]
#          arch: arm64
#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
#          sdkroot: macosx11.3

    steps:
    - name: Checkout PyYAML
      uses: actions/checkout@v2

    - name: Get cached libyaml state
      id: cached_libyaml
      uses: actions/cache@v2
      with:
        path: libyaml
        key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}}

    - name: Ensure libyaml fetched
      run: exit 1
      if: steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: Build/Test/Package
      env:
        C_INCLUDE_PATH: libyaml/include
        CIBW_BUILD: ${{matrix.spec}}
        CIBW_BUILD_VERBOSITY: 1
        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
        LIBRARY_PATH: libyaml/src/.libs
        MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
        SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
      run: |
        python3 -V
        python3 -m pip install -U --user cibuildwheel
        python3 -m cibuildwheel --platform auto --output-dir dist .

    - name: Upload artifacts
      uses: actions/upload-artifact@v2
      with:
        name: dist
        path: dist/*.whl
        if-no-files-found: error

  windows_libyaml:
    name: libyaml ${{matrix.platform}} ${{matrix.arch}}
    runs-on: ${{matrix.platform}}
    strategy:
      matrix:
        include:
        - platform: windows-2019
          arch: x64
        - platform: windows-2019
          arch: win32
    steps:
    - name: Get cached libyaml state
      id: cached_libyaml
      uses: actions/cache@v2
      with:
        path: libyaml
        key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}

    - name: Build libyaml
      shell: bash
      if: steps.cached_libyaml.outputs.cache-hit != 'true'
      run: |
        # git spews all over stderr unless we tell it not to
        export GIT_REDIRECT_STDERR="2>&1"

        if [[ ! -d ./libyaml ]]; then
          git clone -b ${{ env.LIBYAML_REF }} ${{ env.LIBYAML_REPO }} 2>&1
        fi

        pushd libyaml
        git clean -fdx
        popd

        mkdir libyaml/build

        pushd libyaml/build
        cmake.exe -G "Visual Studio 16 2019" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml ..
        cmake.exe --build . --config Release
        popd


  windows_pyyaml:
    needs: windows_libyaml
    name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}}
    runs-on: ${{matrix.platform}}
    strategy:
      matrix:
        include:
        - platform: windows-2019
          build_arch: x64
          python_arch: x64
          spec: 3.6
#        - platform: windows-2019
#          build_arch: x64
#          python_arch: x64
#          spec: 3.7
#        - platform: windows-2019
#          build_arch: x64
#          python_arch: x64
#          spec: 3.8
#        - platform: windows-2019
#          build_arch: x64
#          python_arch: x64
#          spec: 3.9
#        - platform: windows-2019
#          build_arch: x64
#          python_arch: x64
#          spec: '3.10'
        - platform: windows-2019
          build_arch: x64
          python_arch: x64
          spec: '3.11.0-rc.2'
        - platform: windows-2019
          build_arch: win32
          python_arch: x86
          spec: 3.6
#        - platform: windows-2019
#          build_arch: win32
#          python_arch: x86
#          spec: 3.7
#        - platform: windows-2019
#          build_arch: win32
#          python_arch: x86
#          spec: 3.8
#        - platform: windows-2019
#          build_arch: win32
#          python_arch: x86
#          spec: 3.9
#        - platform: windows-2019
#          build_arch: win32
#          python_arch: x86
#          spec: '3.10'
        - platform: windows-2019
          build_arch: win32
          python_arch: x86
          spec: '3.11.0-rc.2'
    steps:
    # autocrlf screws up tests under Windows
    - name: Set git to use LF
      run: |
        git config --global core.autocrlf false
        git config --global core.eol lf

    - name: Checkout pyyaml
      uses: actions/checkout@v2

    - name: Get cached libyaml state
      id: cached_libyaml
      uses: actions/cache@v2
      with:
        path: libyaml
        key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}}

    - name: Ensure libyaml fetched
      run: exit 1
      if: steps.cached_libyaml.outputs.cache-hit != 'true'

    - name: Install python ${{ matrix.spec }}
      uses: actions/setup-python@v2
      with:
        architecture: ${{ matrix.python_arch }}
        python-version: ${{ matrix.spec }}

    - name: Build/Test/Package
      env:
        PYYAML_FORCE_CYTHON: 1
        PYYAML_FORCE_LIBYAML: 1
      shell: bash
      run: |
        set -eux
        python -V
        python -m pip install Cython wheel

        python setup.py \
        --with-libyaml build_ext \
        -I libyaml/include \
        -L libyaml/build/Release \
        -D YAML_DECLARE_STATIC \
        build bdist_wheel

        # run tests on built wheel
        python -m pip install dist/*.whl
        python tests/lib/test_all.py

    - name: Upload artifacts
      uses: actions/upload-artifact@v2
      with:
        name: dist
        path: dist/*.whl
        if-no-files-found: error
...