From 750d7f964d49664d9ef0d68c7deeec230eebc2ff Mon Sep 17 00:00:00 2001 From: Mark Towers Date: Mon, 20 May 2024 15:48:05 +0100 Subject: [PATCH] Add testing for wheels (#527) --- .../actions/download-release-asset/Dockerfile | 13 - .../actions/download-release-asset/action.yml | 15 -- .../download-release-asset/download-asset.sh | 35 --- .github/workflows/ci.yml | 254 +++++++++++++++--- .github/workflows/{wheels.yml => pypi.yml} | 64 +---- .github/workflows/release.yml | 27 -- setup.py | 24 -- tests/python/test_atari_env.py | 22 ++ 8 files changed, 249 insertions(+), 205 deletions(-) delete mode 100644 .github/actions/download-release-asset/Dockerfile delete mode 100644 .github/actions/download-release-asset/action.yml delete mode 100755 .github/actions/download-release-asset/download-asset.sh rename .github/workflows/{wheels.yml => pypi.yml} (61%) delete mode 100644 .github/workflows/release.yml diff --git a/.github/actions/download-release-asset/Dockerfile b/.github/actions/download-release-asset/Dockerfile deleted file mode 100644 index acd7e8936..000000000 --- a/.github/actions/download-release-asset/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM alpine:latest - -RUN apk add --no-cache \ - bash \ - ca-certificates \ - curl \ - wget \ - jq - -COPY download-asset.sh /usr/bin/download-asset -RUN chmod +x /usr/bin/download-asset - -ENTRYPOINT ["/usr/bin/download-asset"] diff --git a/.github/actions/download-release-asset/action.yml b/.github/actions/download-release-asset/action.yml deleted file mode 100644 index 1804abf73..000000000 --- a/.github/actions/download-release-asset/action.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -name: download-release-asset -description: Download release asset from Github release -inputs: - filename: - description: Filename of asset to download - required: true -branding: - color: yellow - icon: download-cloud -runs: - using: docker - image: Dockerfile - args: - - ${{ inputs.filename }} diff --git a/.github/actions/download-release-asset/download-asset.sh b/.github/actions/download-release-asset/download-asset.sh deleted file mode 100755 index 056a21181..000000000 --- a/.github/actions/download-release-asset/download-asset.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Modified from https://github.com/wyozi/download-gh-release-asset -# -set -e -set -o pipefail - -# Ensure that the GITHUB_TOKEN secret is included -if [[ -z "${GITHUB_TOKEN}" ]]; then - echo "Set the GITHUB_TOKEN env variable." - exit 1 -fi - -# Ensure that the file path is present -if [[ -z "${INPUT_FILENAME}" ]]; then - echo "It seems you forgot to pass the filename of the asset to download" - exit 1 -fi - -FILE="${INPUT_FILENAME}" - -asset_id=$(jq ".release.assets | map(select(.name == \"$FILE\"))[0].id" $GITHUB_EVENT_PATH) -if [ "$asset_id" = "null" ]; then - echo "ERROR: asset id not found" - exit 1 -fi; - -AUTH_HEADER="Authorization: token ${GITHUB_TOKEN}" -ASSET_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${asset_id}" - -curl \ - -L \ - -H "${AUTH_HEADER}" \ - -H "Accept:application/octet-stream" \ - -o "${FILE}" \ - "${ASSET_URL}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c6df981d..eabf924f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,9 @@ +# this file includes 4 stages +# 1. pre-commit - runs pre-commit using `.pre-commit-config.yaml` +# 2. build-scratch - (if pre-commit pass) then we build the project from scratch (single python versions) and pytest +# 3. build-wheels - (if pre-commit pass) then we build all the wheels for all valid versions +# 4. test-wheels - (if build-scratch and build-wheel passes) then we install wheel and run pytest for all python versions + name: CI on: workflow_dispatch: @@ -8,12 +14,8 @@ on: required: false default: false push: - tags-ignore: - - "*.*" paths-ignore: - "docs/**" - branches: - - "*" pull_request: branches: - "*" @@ -28,7 +30,7 @@ jobs: python-version: '3.10' - uses: pre-commit/action@v3.0.1 - tests: + build-scratch: name: ${{ matrix.runs-on }} • py${{ matrix.python }} needs: pre-commit defaults: @@ -38,53 +40,54 @@ jobs: fail-fast: false matrix: include: + # To minimise the computational resources, we only use a single python version and the final test-wheels for all python versions - runs-on: ubuntu-latest python: '3.8' triplet: x64-linux-mixed - - runs-on: ubuntu-latest - python: '3.9' - triplet: x64-linux-mixed - - runs-on: ubuntu-latest - python: '3.10' - triplet: x64-linux-mixed - - runs-on: ubuntu-latest - python: '3.11' - triplet: x64-linux-mixed - - runs-on: ubuntu-latest - python: '3.12' - triplet: x64-linux-mixed +# - runs-on: ubuntu-latest +# python: '3.9' +# triplet: x64-linux-mixed +# - runs-on: ubuntu-latest +# python: '3.10' +# triplet: x64-linux-mixed +# - runs-on: ubuntu-latest +# python: '3.11' +# triplet: x64-linux-mixed +# - runs-on: ubuntu-latest +# python: '3.12' +# triplet: x64-linux-mixed - runs-on: macos-12 # macos latest is macos-14-arm64 running on M1 chips which doesn't have python support python: '3.8' triplet: x64-osx-mixed - - runs-on: macos-12 - python: '3.9' - triplet: x64-osx-mixed - - runs-on: macos-12 - python: '3.10' - triplet: x64-osx-mixed - - runs-on: macos-12 - python: '3.11' - triplet: x64-osx-mixed - - runs-on: macos-12 - python: '3.12' - triplet: x64-osx-mixed +# - runs-on: macos-12 +# python: '3.9' +# triplet: x64-osx-mixed +# - runs-on: macos-12 +# python: '3.10' +# triplet: x64-osx-mixed +# - runs-on: macos-12 +# python: '3.11' +# triplet: x64-osx-mixed +# - runs-on: macos-12 +# python: '3.12' +# triplet: x64-osx-mixed - runs-on: windows-latest python: '3.8' triplet: x64-windows - - runs-on: windows-latest - python: '3.9' - triplet: x64-windows - - runs-on: windows-latest - python: '3.10' - triplet: x64-windows - - runs-on: windows-latest - python: '3.11' - triplet: x64-windows - - runs-on: windows-latest - python: '3.12' - triplet: x64-windows +# - runs-on: windows-latest +# python: '3.9' +# triplet: x64-windows +# - runs-on: windows-latest +# python: '3.10' +# triplet: x64-windows +# - runs-on: windows-latest +# python: '3.11' +# triplet: x64-windows +# - runs-on: windows-latest +# python: '3.12' +# triplet: x64-windows env: VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} runs-on: ${{ matrix.runs-on }} @@ -123,3 +126,172 @@ jobs: - name: Test run: python -m pytest + + build-wheels: + name: ${{ matrix.runs-on }} • ${{ matrix.arch }} + needs: pre-commit + defaults: + run: + shell: bash + strategy: + matrix: + include: + - runs-on: ubuntu-latest + arch: x86_64 + - runs-on: windows-latest + arch: AMD64 + - runs-on: macos-12 + arch: x86_64 + - runs-on: macos-12 + arch: arm64 + runs-on: ${{ matrix.runs-on }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + if: runner.os == 'linux' + id: buildx + uses: docker/setup-buildx-action@v1 + with: + install: true + + - name: Build Docker image with vcpkg + if: runner.os == 'linux' + # using build-push-action (without push) to make use of cache arguments + uses: docker/build-push-action@v2 + with: + context: . + file: .github/docker/manylinux-vcpkg.Dockerfile + tags: manylinux-vcpkg:latest + push: false + load: true + + - uses: microsoft/setup-msbuild@v2 + if: runner.os == 'Windows' + # TODO(jfarebro): 02/16/2023 - There's a bug where pkg-config isn't installed on the macOS + # runner. See: https://github.com/actions/runner-images/pull/7125 + - name: Install pkg-config on macOS + if: runner.os == 'macOS' + run: brew install pkg-config + + - uses: lukka/run-vcpkg@v11 + with: + vcpkgGitCommitId: "8150939b69720adc475461978e07c2d2bf5fb76e" + # There's a permissions issue with the cache + # https://github.com/microsoft/vcpkg/issues/20121 + doNotCache: true + + - name: Download and unpack ROMs + run: ./scripts/download_unpack_roms.sh + + - name: Build wheels + uses: pypa/cibuildwheel@v2.17.0 + env: + CIBW_ARCHS: "${{ matrix.arch }}" + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/*.whl + + test-wheels: + name: Test wheels + needs: [build-wheels, build-scratch] + + strategy: + matrix: + include: + # example wheel names (if the wheel names change, look at the `ls wheels/` for the new names) + # ale_py-0.9.0-cp310-cp310-macosx_10_15_x86_64.whl + # ale_py-0.9.0-cp310-cp310-macosx_11_0_arm64.whl + # ale_py-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + # ale_py-0.9.0-cp310-cp310-win_amd64.whl + - runs-on: ubuntu-latest # arch: x86_64 + python: '3.8' + wheel-name: 'cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64' + - runs-on: ubuntu-latest # arch: x86_64 + python: '3.9' + wheel-name: 'cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64' + - runs-on: ubuntu-latest # arch: x86_64 + python: '3.10' + wheel-name: 'cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64' + - runs-on: ubuntu-latest # arch: x86_64 + python: '3.11' + wheel-name: 'cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64' + - runs-on: ubuntu-latest # arch: x86_64 + python: '3.12' + wheel-name: 'cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64' + + - runs-on: windows-latest # arch: AMD64 + python: '3.8' + wheel-name: 'cp38-cp38-win_amd64' + - runs-on: windows-latest # arch: AMD64 + python: '3.9' + wheel-name: 'cp39-cp39-win_amd64' + - runs-on: windows-latest # arch: AMD64 + python: '3.10' + wheel-name: 'cp310-cp310-win_amd64' + - runs-on: windows-latest # arch: AMD64 + python: '3.11' + wheel-name: 'cp311-cp311-win_amd64' + - runs-on: windows-latest # arch: AMD64 + python: '3.12' + wheel-name: 'cp312-cp312-win_amd64' + + - runs-on: macos-12 # arch: x86_64 + python: '3.8' + wheel-name: 'cp38-cp38-macosx_10_15_x86_64' + - runs-on: macos-12 # arch: x86_64 + python: '3.9' + wheel-name: 'cp39-cp39-macosx_10_15_x86_64' + - runs-on: macos-12 # arch: x86_64 + python: '3.10' + wheel-name: 'cp310-cp310-macosx_10_15_x86_64' + - runs-on: macos-12 # arch: x86_64 + python: '3.11' + wheel-name: 'cp311-cp311-macosx_10_15_x86_64' + - runs-on: macos-12 # arch: x86_64 + python: '3.12' + wheel-name: 'cp312-cp312-macosx_10_15_x86_64' + + - runs-on: macos-14 # arch: arm64 + python: '3.8' + wheel-name: 'cp38-cp38-macosx_11_0_arm64' + - runs-on: macos-14 # arch: arm64 + python: '3.9' + wheel-name: 'cp39-cp39-macosx_11_0_arm64' + - runs-on: macos-14 # arch: arm64 + python: '3.10' + wheel-name: 'cp310-cp310-macosx_11_0_arm64' + - runs-on: macos-14 # arch: arm64 + python: '3.11' + wheel-name: 'cp311-cp311-macosx_11_0_arm64' + - runs-on: macos-14 # arch: arm64 + python: '3.12' + wheel-name: 'cp312-cp312-macosx_11_0_arm64' + + runs-on: ${{ matrix.runs-on }} + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - uses: actions/download-artifact@v1 + with: + name: wheels + + - run: ls wheels/ + + - name: Build + # wildcarding doesn't work for some reason, therefore, update the project version here + run: python -m pip install wheels/ale_py-0.9.0-${{ matrix.wheel-name }}.whl + + - name: Install Gymnasium and pytest + run: python -m pip install gymnasium>=1.0.0a1 pytest + + - name: Test + run: python -m pytest diff --git a/.github/workflows/wheels.yml b/.github/workflows/pypi.yml similarity index 61% rename from .github/workflows/wheels.yml rename to .github/workflows/pypi.yml index bcbbd5385..1d65daad1 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/pypi.yml @@ -1,12 +1,15 @@ -name: Wheels +# workflow on publish +# 1. build the wheels for relevant systems +# 2. download artifacts and upload to PyPI + +name: Push wheels to pypi on: - push: - tags: - - 'v*' + release: + types: [ published ] jobs: - build: + build-wheels: name: ${{ matrix.runs-on }} • ${{ matrix.arch }} defaults: run: @@ -23,6 +26,7 @@ jobs: - runs-on: macos-12 arch: arm64 runs-on: ${{ matrix.runs-on }} + steps: - uses: actions/checkout@v4 @@ -73,62 +77,22 @@ jobs: name: wheels path: ./wheelhouse/*.whl - pypi: - name: Deploy wheels to PyPi test + push-pypi: + name: Deploy wheels to PyPi runs-on: ubuntu-latest - needs: build + needs: build-wheels permissions: id-token: write + steps: - uses: actions/download-artifact@v1 with: name: wheels - - name: Append PyPi test SHA - run: | - find wheels -type f -exec sh -c \ - "mv {} \$(echo {} | awk -F\"-\" '{OFS = FS; \$2 = \$2\"+${GITHUB_SHA::7}\"; print}')" \; - - name: Publish to PyPi test uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ - password: ${{ secrets.PYPI_TEST_TOKEN }} - repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.PYPI_TOKEN }} packages-dir: wheels/ print-hash: true - - - release: - name: Stage wheels to Github releases - runs-on: ubuntu-latest - needs: build - steps: - - uses: actions/download-artifact@v1 - with: - name: wheels - - - name: Create wheel archive - run: | - zip --junk-paths wheels wheels/* - - - name: Create release - id: create_release - uses: actions/create-release@v1.0.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: true - prerelease: ${{ contains(github.ref, '-') }} - - - name: Upload artifacts to release - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./wheels.zip - asset_name: wheels.zip - asset_content_type: application/zip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 04aae2f98..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Release - -on: - release: - types: [published] - -jobs: - wheels: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: ./.github/actions/download-release-asset - name: Download release wheels - with: - filename: wheels.zip - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Unzip wheels asset - run: | - unzip wheels.zip -d dist - - name: Publish to PyPi - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} - print_hash: true diff --git a/setup.py b/setup.py index 7307c30bf..3bd65e421 100644 --- a/setup.py +++ b/setup.py @@ -127,30 +127,6 @@ def parse_version(version_file): version = fp.read().strip() assert semver_prog.match(version) is not None - if os.getenv("CIBUILDWHEEL") is not None: - ci_ref = os.getenv("GITHUB_REF") - assert ci_ref is not None, "Github ref not found, are we running in CI?" - - ci_version_match = semver_prog.search(ci_ref) - assert ci_version_match is not None, f"Couldn't match semver in {ci_ref}" - - ci_version = ci_version_match.group(0) - assert ci_version.startswith( - version - ), f"{ci_version} not prefixed with {version}" - - version = ci_version - else: - sha = ( - subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], capture_output=True, cwd=here - ) - .stdout.decode("ascii") - .strip() - ) - if sha: - version += f"+{sha}" - return version diff --git a/tests/python/test_atari_env.py b/tests/python/test_atari_env.py index e146132f7..1521cc01b 100644 --- a/tests/python/test_atari_env.py +++ b/tests/python/test_atari_env.py @@ -11,6 +11,28 @@ from utils import test_rom_path, tetris_env # noqa: F401 +def test_roms_register(): + registered_roms = [ + env_id + for env_id, spec in gymnasium.registry.items() + if spec.entry_point == "ale_py.env:AtariEnv" + ] + + registered_v0_roms = list(filter(lambda env_id: "v0" in env_id, registered_roms)) + registered_v4_roms = list(filter(lambda env_id: "v4" in env_id, registered_roms)) + registered_v5_roms = list(filter(lambda env_id: "v5" in env_id, registered_roms)) + + assert ( + len(registered_v0_roms) == 372 + ), f"{len(registered_roms)}, {len(registered_v0_roms)}, {len(registered_v4_roms)}, {len(registered_v5_roms)}" + assert len(registered_v4_roms) == 372 + assert len(registered_v5_roms) == 216 + + assert len(registered_roms) == len(registered_v0_roms) + len( + registered_v4_roms + ) + len(registered_v5_roms) + + @pytest.mark.parametrize( "env_id", [