Add testing for wheels (#527)

.github/workflows/ci.yml

@@ -1,3 +1,9 @@
+# this file includes 4 stages
+# 1. pre-commit - runs pre-commit using `.pre-commit-config.yaml`
+# 2. build-scratch - (if pre-commit pass) then we build the project from scratch (single python versions) and pytest
+# 3. build-wheels - (if pre-commit pass) then we build all the wheels for all valid versions
+# 4. test-wheels - (if build-scratch and build-wheel passes) then we install wheel and run pytest for all python versions
+
 name: CI
 on:
   workflow_dispatch:
@@ -8,12 +14,8 @@ on:
         required: false
         default: false
   push:
-    tags-ignore:
-      - "*.*"
     paths-ignore:
       - "docs/**"
-    branches:
-      - "*"
   pull_request:
     branches:
       - "*"
@@ -28,7 +30,7 @@ jobs:
         python-version: '3.10'
     - uses: pre-commit/[email protected]
 
-  tests:
+  build-scratch:
     name: ${{ matrix.runs-on }} • py${{ matrix.python }}
     needs: pre-commit
     defaults:
@@ -38,53 +40,54 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # To minimise the computational resources, we only use a single python version and the final test-wheels for all python versions
           - runs-on: ubuntu-latest
             python: '3.8'
             triplet: x64-linux-mixed
-          - runs-on: ubuntu-latest
-            python: '3.9'
-            triplet: x64-linux-mixed
-          - runs-on: ubuntu-latest
-            python: '3.10'
-            triplet: x64-linux-mixed
-          - runs-on: ubuntu-latest
-            python: '3.11'
-            triplet: x64-linux-mixed
-          - runs-on: ubuntu-latest
-            python: '3.12'
-            triplet: x64-linux-mixed
+#          - runs-on: ubuntu-latest
+#            python: '3.9'
+#            triplet: x64-linux-mixed
+#          - runs-on: ubuntu-latest
+#            python: '3.10'
+#            triplet: x64-linux-mixed
+#          - runs-on: ubuntu-latest
+#            python: '3.11'
+#            triplet: x64-linux-mixed
+#          - runs-on: ubuntu-latest
+#            python: '3.12'
+#            triplet: x64-linux-mixed
 
           - runs-on: macos-12  # macos latest is macos-14-arm64 running on M1 chips which doesn't have python support
             python: '3.8'
             triplet: x64-osx-mixed
-          - runs-on: macos-12
-            python: '3.9'
-            triplet: x64-osx-mixed
-          - runs-on: macos-12
-            python: '3.10'
-            triplet: x64-osx-mixed
-          - runs-on: macos-12
-            python: '3.11'
-            triplet: x64-osx-mixed
-          - runs-on: macos-12
-            python: '3.12'
-            triplet: x64-osx-mixed
+#          - runs-on: macos-12
+#            python: '3.9'
+#            triplet: x64-osx-mixed
+#          - runs-on: macos-12
+#            python: '3.10'
+#            triplet: x64-osx-mixed
+#          - runs-on: macos-12
+#            python: '3.11'
+#            triplet: x64-osx-mixed
+#          - runs-on: macos-12
+#            python: '3.12'
+#            triplet: x64-osx-mixed
 
           - runs-on: windows-latest
             python: '3.8'
             triplet: x64-windows
-          - runs-on: windows-latest
-            python: '3.9'
-            triplet: x64-windows
-          - runs-on: windows-latest
-            python: '3.10'
-            triplet: x64-windows
-          - runs-on: windows-latest
-            python: '3.11'
-            triplet: x64-windows
-          - runs-on: windows-latest
-            python: '3.12'
-            triplet: x64-windows
+#          - runs-on: windows-latest
+#            python: '3.9'
+#            triplet: x64-windows
+#          - runs-on: windows-latest
+#            python: '3.10'
+#            triplet: x64-windows
+#          - runs-on: windows-latest
+#            python: '3.11'
+#            triplet: x64-windows
+#          - runs-on: windows-latest
+#            python: '3.12'
+#            triplet: x64-windows
     env:
       VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }}
     runs-on: ${{ matrix.runs-on }}
@@ -123,3 +126,172 @@ jobs:
 
     - name: Test
       run: python -m pytest
+
+  build-wheels:
+    name: ${{ matrix.runs-on }} • ${{ matrix.arch }}
+    needs: pre-commit
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - runs-on: ubuntu-latest
+            arch: x86_64
+          - runs-on: windows-latest
+            arch: AMD64
+          - runs-on: macos-12
+            arch: x86_64
+          - runs-on: macos-12
+            arch: arm64
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        if: runner.os == 'linux'
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Build Docker image with vcpkg
+        if: runner.os == 'linux'
+        # using build-push-action (without push) to make use of cache arguments
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: .github/docker/manylinux-vcpkg.Dockerfile
+          tags: manylinux-vcpkg:latest
+          push: false
+          load: true
+
+      - uses: microsoft/setup-msbuild@v2
+        if: runner.os == 'Windows'
+      # TODO(jfarebro): 02/16/2023 - There's a bug where pkg-config isn't installed on the macOS
+      # runner. See: https://github.com/actions/runner-images/pull/7125
+      - name: Install pkg-config on macOS
+        if: runner.os == 'macOS'
+        run: brew install pkg-config
+
+      - uses: lukka/run-vcpkg@v11
+        with:
+          vcpkgGitCommitId: "8150939b69720adc475461978e07c2d2bf5fb76e"
+          # There's a permissions issue with the cache
+          # https://github.com/microsoft/vcpkg/issues/20121
+          doNotCache: true
+
+      - name: Download and unpack ROMs
+        run: ./scripts/download_unpack_roms.sh
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_ARCHS: "${{ matrix.arch }}"
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
+
+  test-wheels:
+    name: Test wheels
+    needs: [build-wheels, build-scratch]
+
+    strategy:
+      matrix:
+        include:
+          # example wheel names (if the wheel names change, look at the `ls wheels/` for the new names)
+          # ale_py-0.9.0-cp310-cp310-macosx_10_15_x86_64.whl
+          # ale_py-0.9.0-cp310-cp310-macosx_11_0_arm64.whl
+          # ale_py-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+          # ale_py-0.9.0-cp310-cp310-win_amd64.whl
+          - runs-on: ubuntu-latest  # arch: x86_64
+            python: '3.8'
+            wheel-name: 'cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64'
+          - runs-on: ubuntu-latest  # arch: x86_64
+            python: '3.9'
+            wheel-name: 'cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64'
+          - runs-on: ubuntu-latest  # arch: x86_64
+            python: '3.10'
+            wheel-name: 'cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64'
+          - runs-on: ubuntu-latest  # arch: x86_64
+            python: '3.11'
+            wheel-name: 'cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64'
+          - runs-on: ubuntu-latest  # arch: x86_64
+            python: '3.12'
+            wheel-name: 'cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64'
+
+          - runs-on: windows-latest  # arch: AMD64
+            python: '3.8'
+            wheel-name: 'cp38-cp38-win_amd64'
+          - runs-on: windows-latest  # arch: AMD64
+            python: '3.9'
+            wheel-name: 'cp39-cp39-win_amd64'
+          - runs-on: windows-latest  # arch: AMD64
+            python: '3.10'
+            wheel-name: 'cp310-cp310-win_amd64'
+          - runs-on: windows-latest  # arch: AMD64
+            python: '3.11'
+            wheel-name: 'cp311-cp311-win_amd64'
+          - runs-on: windows-latest  # arch: AMD64
+            python: '3.12'
+            wheel-name: 'cp312-cp312-win_amd64'
+
+          - runs-on: macos-12  # arch: x86_64
+            python: '3.8'
+            wheel-name: 'cp38-cp38-macosx_10_15_x86_64'
+          - runs-on: macos-12  # arch: x86_64
+            python: '3.9'
+            wheel-name: 'cp39-cp39-macosx_10_15_x86_64'
+          - runs-on: macos-12  # arch: x86_64
+            python: '3.10'
+            wheel-name: 'cp310-cp310-macosx_10_15_x86_64'
+          - runs-on: macos-12  # arch: x86_64
+            python: '3.11'
+            wheel-name: 'cp311-cp311-macosx_10_15_x86_64'
+          - runs-on: macos-12  # arch: x86_64
+            python: '3.12'
+            wheel-name: 'cp312-cp312-macosx_10_15_x86_64'
+
+          - runs-on: macos-14  # arch: arm64
+            python: '3.8'
+            wheel-name: 'cp38-cp38-macosx_11_0_arm64'
+          - runs-on: macos-14  # arch: arm64
+            python: '3.9'
+            wheel-name: 'cp39-cp39-macosx_11_0_arm64'
+          - runs-on: macos-14  # arch: arm64
+            python: '3.10'
+            wheel-name: 'cp310-cp310-macosx_11_0_arm64'
+          - runs-on: macos-14  # arch: arm64
+            python: '3.11'
+            wheel-name: 'cp311-cp311-macosx_11_0_arm64'
+          - runs-on: macos-14  # arch: arm64
+            python: '3.12'
+            wheel-name: 'cp312-cp312-macosx_11_0_arm64'
+
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: wheels
+
+      - run: ls wheels/
+
+      - name: Build
+        # wildcarding doesn't work for some reason, therefore, update the project version here
+        run: python -m pip install wheels/ale_py-0.9.0-${{ matrix.wheel-name }}.whl
+
+      - name: Install Gymnasium and pytest
+        run: python -m pip install gymnasium>=1.0.0a1 pytest
+
+      - name: Test
+        run: python -m pytest