Test suite clean up

.github/workflows/build.yml

@@ -83,15 +83,20 @@ jobs:
             --install defcon \
             --install gadopt \
             --install asQ \
+            --package-branch PyOP2 JDBetteridge/remove_comm_hash \
             || (cat firedrake-install.log && /bin/false)
       - name: Install test dependencies
         run: |
           . ../firedrake_venv/bin/activate
+          cd ../firedrake_venv/src/pytest-mpi
+          git checkout JDBetteridge/spawn
+          cd -
           python "$(which firedrake-clean)"
           python -m pip install \
-            pytest-xdist pytest-timeout ipympl
+            pytest-xdist pytest-timeout ipympl pytest-split
+          pip install git+https://github.com/JDBetteridge/mpispawn
           python -m pip list
-      - name: Test Firedrake
+      - name: Test Firedrake Serial
         run: |
           . ../firedrake_venv/bin/activate
           echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
@@ -102,15 +107,103 @@ jobs:
             --timeout=1800 \
             --timeout-method=thread \
             -o faulthandler_timeout=1860 \
+            -m "not parallel" \
             -n 12 --dist worksteal \
-            --junit-xml=firedrake.xml \
+            --junit-xml=firedrake1.xml \
             -sv tests
-        timeout-minutes: 120
+        timeout-minutes: 60
+      - name: Test Firedrake 2 ranks
+        run: |
+          . ../firedrake_venv/bin/activate
+          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
+          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
+          mpispawn -nU 12 -nW 2 --propagate-errcodes pytest \
+            --splitting-algorithm least_duration \
+            --splits \$MPISPAWN_NUM_TASKS \
+            --group \$MPISPAWN_TASK_ID1 \
+            --timeout=1800 \
+            --timeout-method=thread \
+            -o faulthandler_timeout=1860 \
+            --junit-xml=firedrake2_\$MPISPAWN_TASK_ID1.xml \
+            -m "parallel[\$MPISPAWN_WORLD_SIZE] and not broken" \
+            -v tests
+        timeout-minutes: 30
+      - name: Test Firedrake 3 ranks
+        run: |
+          . ../firedrake_venv/bin/activate
+          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
+          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
+          mpispawn -nU 12 -nW 3 --propagate-errcodes pytest \
+            --splitting-algorithm least_duration \
+            --splits \$MPISPAWN_NUM_TASKS \
+            --group \$MPISPAWN_TASK_ID1 \
+            --timeout=1800 \
+            --timeout-method=thread \
+            -o faulthandler_timeout=1860 \
+            --junit-xml=firedrake3_\$MPISPAWN_TASK_ID1.xml \
+            -m "parallel[\$MPISPAWN_WORLD_SIZE] and not broken" \
+            -v tests
+        timeout-minutes: 60
+      - name: Test Firedrake 4 ranks
+        run: |
+          . ../firedrake_venv/bin/activate
+          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
+          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
+          mpispawn -nU 8 -nW 4 --propagate-errcodes pytest \
+            --splitting-algorithm least_duration \
+            --splits \$MPISPAWN_NUM_TASKS \
+            --group \$MPISPAWN_TASK_ID1 \
+            --timeout=1800 \
+            --timeout-method=thread \
+            -o faulthandler_timeout=1860 \
+            --junit-xml=firedrake4_\$MPISPAWN_TASK_ID1.xml \
+            -m "parallel[\$MPISPAWN_WORLD_SIZE] and not broken" \
+            -v tests
+        timeout-minutes: 30
+      - name: Test Firedrake 6 ranks
+        run: |
+          . ../firedrake_venv/bin/activate
+          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
+          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
+          mpiexec -n 6 python -m pytest \
+            --timeout=1800 \
+            --timeout-method=thread \
+            -o faulthandler_timeout=1860 \
+            --junit-xml=firedrake6.xml \
+            -m "parallel[6] and not broken" \
+            -sv tests
+        timeout-minutes: 30
+      - name: Test Firedrake 7 ranks
+        run: |
+          . ../firedrake_venv/bin/activate
+          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
+          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
+          mpiexec -n 7 python -m pytest \
+            --timeout=1800 \
+            --timeout-method=thread \
+            -o faulthandler_timeout=1860 \
+            --junit-xml=firedrake7.xml \
+            -m "parallel[7] and not broken" \
+            -sv tests
+        timeout-minutes: 30
+      - name: Test Firedrake 8 ranks
+        run: |
+          . ../firedrake_venv/bin/activate
+          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
+          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
+          mpiexec -n 8 python -m pytest \
+            --timeout=1800 \
+            --timeout-method=thread \
+            -o faulthandler_timeout=1860 \
+            --junit-xml=firedrake8.xml \
+            -m "parallel[8] and not broken" \
+            -sv tests
+        timeout-minutes: 30
       - name: Publish Test Report
         uses: mikepenz/[email protected]
         if: ${{ always() && ( github.ref != 'refs/heads/master') }}
         with:
-          report_paths: '/__w/firedrake/firedrake/firedrake.xml'
+          report_paths: '/__w/firedrake/firedrake/firedrake*.xml'
           comment: true
           check_name: "Firedrake ${{ matrix.scalar-type }}"
           updateComment: true
@@ -125,6 +218,7 @@ jobs:
             --timeout=600 \
             --timeout-method=thread \
             -o faulthandler_timeout=660 \
+            -m "not slow" \
             -n 12 --dist worksteal \
             -sv tests/firedrake_adjoint
         timeout-minutes: 30