Initial commit mpi tests

PennyLaneAI · Oct 13, 2023 · f5f5b3f · f5f5b3f
1 parent 4edb007
commit f5f5b3f
Show file tree

Hide file tree

Showing 14 changed files with 597 additions and 12 deletions.
diff --git a/.github/workflows/changelog_reminder.yml b/.github/workflows/changelog_reminder.yml
@@ -1,5 +1,5 @@
 on:
-  pull_request:
+  #pull_request:
     types: [opened, ready_for_review]
 
 name: Changelog Reminder

diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -1,6 +1,6 @@
 name: Formatting check
 on:
-  pull_request:
+  #pull_request:
   push:
     branches:
       - master

diff --git a/.github/workflows/test_gpu_cu11.yml b/.github/workflows/test_gpu_cu11.yml
@@ -1,6 +1,6 @@
 name: Testing::Linux::x86_64 (Lightning-GPU)
 on:
-  pull_request:
+  #pull_request:
   push:
     branches:
       - master

diff --git a/.github/workflows/tests_linux.yml b/.github/workflows/tests_linux.yml
@@ -10,7 +10,7 @@ on:
         type: string
         required: true
         description: The version of PennyLane to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
-  pull_request:
+  #pull_request:
   push:
     branches:
       - master

diff --git a/.github/workflows/tests_linux_x86_mpich.yml b/.github/workflows/tests_linux_x86_mpich.yml
@@ -0,0 +1,306 @@
+name: Tests::Linux::x86_64_MPICH
+on:
+  workflow_call:
+    inputs:
+      lightning-gpu-version:
+        type: string
+        required: true
+        description: The version of lightning to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
+      pennylane-version:
+        type: string
+        required: true
+        description: The version of PennyLane to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
+  release:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  COVERAGE_FLAGS: "--cov=pennylane_lightning_gpu --cov-report=term-missing --cov-report=xml:./coverage.xml --no-flaky-report -p no:warnings --tb=native" 
+  GCC_VERSION: 11
+  OMP_NUM_THREADS: "2"
+  CI_CUDA_ARCH: 86
+
+concurrency:
+  group: gpu-test-mpich-${{ github.ref }}-${{ inputs.lightning-gpu-version }}-${{ inputs.pennylane-version }}
+  cancel-in-progress: true
+
+jobs:
+  cpp_mpich_tests:
+    if: contains(github.event.pull_request.labels.*.name, 'ci:use-multi-gpu-runner') || (inputs.lightning-gpu-version != '' && inputs.pennylane-version != '')
+
+    runs-on:
+      - self-hosted
+      - linux
+      - x64
+      - ubuntu-22.04
+      - multi-gpu
+
+    strategy:
+      max-parallel: 1
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          # Checkout entire git-history if workflow_call passes 'stable' as we need to find the most recent git-tag
+          fetch-depth: ${{ inputs.lightning-gpu-version == 'stable' && 0 || 1 }}
+
+      - name: Switch to stable build of Lightning-GPU
+        if: inputs.lightning-gpu-version == 'stable'
+        run: git checkout $(git tag | sort -V | tail -1)
+
+      - uses: actions/setup-python@v4
+        id: setup_python
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      # Since the self-hosted runner can be re-used. It is best to set up all package
+      # installations in a virtual environment that gets cleaned at the end of each workflow run
+      - name: Setup Python virtual environment
+        id: setup_venv
+        env:
+          VENV_NAME: ${{ github.workspace }}/venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
+        run: |
+          # Clear any pre-existing venvs
+          rm -rf venv_*
+
+          # Create new venv for this workflow_run
+          python --version
+          python -m venv ${{ env.VENV_NAME }}
+
+          # Add the venv to PATH for subsequent steps
+          echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
+
+          # Adding venv name as an output for subsequent steps to reference if needed
+          echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
+
+      - name: Display Python-Path
+        id: python_path
+        run: |
+          py_path=$(which python)
+          echo "Python Interpreter Path => $py_path"
+          echo "python=$py_path" >> $GITHUB_OUTPUT
+          
+          pip_path=$(which python)
+          echo "PIP Path => $pip_path"
+          echo "pip=$pip_path" >> $GITHUB_OUTPUT
+
+      - name: Install required packages
+        run: |
+          python -m pip install ninja cmake custatevec-cu11 
+
+      - name: Validate GPU version and installed compiler
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /opt/modules
+          module load cuda/11.8
+          which -a nvcc
+          nvcc --version
+
+      - name: Validate Multi-GPU packages
+        if: contains(github.event.pull_request.labels.*.name, 'ci:use-multi-gpu-runner') || (inputs.lightning-gpu-version != '' && inputs.pennylane-version != '')
+        run: |
+          source /etc/profile.d/modules.sh
+
+          module use /opt/modules/
+
+          echo 'Checking for OpenMPI'
+          module load openmpi
+          mpirun --version
+          which mpicc
+          which mpirun
+          module unload openmpi
+
+          echo 'Checking for MPICH'
+          module load mpich
+          mpiexec --version
+          which mpicc
+          which mpiexec
+          module unload mpich
+
+      - name: Install Latest PennyLane
+        if: inputs.pennylane-version == 'latest'
+        run: python -m pip install git+https://github.com/PennyLaneAI/pennylane.git@master
+
+      - name: Build and run unit tests with MPICH backend
+        run: |
+            source /etc/profile.d/modules.sh
+            module use /opt/modules/
+            module load mpich
+            module unload mpich
+            export PATH=$PATH:/usr/local/cuda/bin:/opt/mpi/mpich/include:/opt/mpi/mpich/bin
+            export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/opt/mpi/mpich/lib:$(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum/lib')")
+            cmake . -BBuild \
+              -DCMAKE_PREFIX_PATH=/opt/mpi/mpich/ \
+              -DPLLGPU_ENABLE_MPI=On \
+              -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+              -DPLLGPU_BUILD_TESTS=ON \
+              -DCMAKE_CXX_COMPILER="$(which g++-${{ env.GCC_VERSION }})" \
+              -DLIGHTNING_RELEASE_TAG="master" \
+              -DCMAKE_CUDA_COMPILER="/usr/local/cuda/bin/nvcc" \
+              -DCMAKE_CUDA_ARCHITECTURES="86" \
+              -DPython_EXECUTABLE:FILE="${{ steps.python_path.outputs.python }}" \
+              -G Ninja
+            cmake --build ./Build
+            cd ./Build
+            mkdir -p ./tests/results
+            /opt/mpi/mpich/bin/mpirun -np 2 ./pennylane_lightning_gpu/src/tests/mpi_runner --order lex --reporter junit --out ./tests/results/report_${{ github.job }}.xml
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ubuntu-tests-reports
+          path: ./Build/tests/results/report_${{ github.job }}.xml
+
+      - name: Build and run unit tests for code coverage for MPICH backend
+        run: |
+            source /etc/profile.d/modules.sh
+            module use /opt/modules/
+            module load mpich
+            module unload mpich
+            export PATH=$PATH:/usr/local/cuda/bin:/opt/mpi/mpich/include:/opt/mpi/mpich/bin 
+            export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/opt/mpi/mpich/lib:$(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum/lib')")
+            cmake . -BBuildCov \
+              -DCMAKE_PREFIX_PATH=/opt/mpi/mpich/ \
+              -DPLLGPU_ENABLE_MPI=On \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DPLLGPU_BUILD_TESTS=ON \
+              -DPLLGPU_ENABLE_COVERAGE=ON \
+              -DCMAKE_CXX_COMPILER="$(which g++-${{ env.GCC_VERSION }})" \
+              -DLIGHTNING_RELEASE_TAG="master" \
+              -DCMAKE_CUDA_COMPILER="/usr/local/cuda/bin/nvcc" \
+              -DCMAKE_CUDA_ARCHITECTURES="86" \
+              -DPython_EXECUTABLE:FILE="${{ steps.python_path.outputs.python }}" \
+              -G Ninja
+            cmake --build ./BuildCov
+            cd ./BuildCov
+            /opt/mpi/mpich/bin/mpirun -np 2 ./pennylane_lightning_gpu/src/tests/mpi_runner
+            lcov --directory . -b ../pennylane_lightning_gpu/src --capture --output-file coverage.info
+            lcov --remove coverage.info '/usr/*' --output-file coverage.info
+            mv coverage.info coverage-${{ github.job }}.info
+
+      - name: Upload code coverage results for MPICH backend
+        uses: actions/upload-artifact@v3
+        with:
+          name: ubuntu-codecov-results-cpp
+          path: ./BuildCov/coverage-${{ github.job }}.info
+
+      - name: Cleanup
+        if: always()
+        run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
+          rm -rf *
+          rm -rf .git
+          rm -rf .gitignore
+          rm -rf .github
+          pip cache purge
+
+
+  python_mpich_tests:
+    if: contains(github.event.pull_request.labels.*.name, 'ci:use-multi-gpu-runner') || (inputs.lightning-gpu-version != '' && inputs.pennylane-version != '')
+
+    runs-on:
+      - self-hosted
+      - linux
+      - x64
+      - ubuntu-22.04
+      - multi-gpu
+
+    strategy:
+      max-parallel: 1
+
+    steps:
+      - name: Checkout pennyLane-lightning-gpu
+        uses: actions/checkout@v3
+        with:
+          # Checkout entire git-history if workflow_call passes 'stable' as we need to find the most recent git-tag
+          fetch-depth: ${{ inputs.lightning-gpu-version == 'stable' && 0 || 1 }}
+
+      - name: Switch to stable build of Lightning-GPU
+        if: inputs.lightning-gpu-version == 'stable'
+        run: git checkout $(git tag | sort -V | tail -1)
+
+      - uses: actions/setup-python@v4
+        id: setup_python
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      # Since the self-hosted runner can be re-used. It is best to set up all package
+      # installations in a virtual environment that gets cleaned at the end of each workflow run
+      - name: Setup Python virtual environment
+        id: setup_venv
+        env:
+          VENV_NAME: ${{ github.workspace }}/venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
+        run: |
+          # Clear any pre-existing venvs
+          rm -rf venv_*
+
+          # Create new venv for this workflow_run
+          python --version
+          python -m venv ${{ env.VENV_NAME }}
+
+          # Add the venv to PATH for subsequent steps
+          echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
+
+          # Adding venv name as an output for subsequent steps to reference if needed
+          source ${{ env.VENV_NAME }}/bin/activate
+          echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
+
+      - name: Display Python-Path
+        id: python_path
+        run: |
+          py_path=$(which python)
+          echo "Python Interpreter Path => $py_path"
+          echo "python=$py_path" >> $GITHUB_OUTPUT
+          
+          pip_path=$(which python)
+          echo "PIP Path => $pip_path"
+          echo "pip=$pip_path" >> $GITHUB_OUTPUT
+
+      - name: Install Latest PennyLane
+        # We want to install the latest PL on non workflow_call events
+        if: inputs.pennylane-version == 'latest'  || inputs.pennylane-version == ''
+        run: python -m pip install git+https://github.com/PennyLaneAI/pennylane.git@master
+
+      - name: Install required packages (MPICH backend)
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /opt/modules/
+          module load mpich
+          module unload mpich
+          export PATH=$PATH:/usr/local/cuda/bin:/opt/mpi/mpich/include:/opt/mpi/mpich/bin 
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/opt/mpi/mpich/lib:$(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum/lib')")
+          python -m pip install pip~=22.0
+          python -m pip install ninja cmake custatevec-cu11 pytest pytest-mock flaky pytest-cov mpi4py
+          # Sync with latest master branches
+          python -m pip install git+https://github.com/PennyLaneAI/pennylane-lightning.git@master --force-reinstall --no-deps
+          
+      - name: Build and install package (MPICH backend)
+        env: 
+          CUQUANTUM_SDK: $(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum/lib')")
+        run: |
+          python setup.py build_ext -i --define="CMAKE_CXX_COMPILER=$(which g++-${{ env.GCC_VERSION }});CMAKE_PREFIX_PATH=/opt/mpi/mpich;PLLGPU_ENABLE_MPI=On;LIGHTNING_RELEASE_TAG=master;CMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc;CMAKE_CUDA_ARCHITECTURES=${{ env.CI_CUDA_ARCH }};Python_EXECUTABLE=${{ steps.python_path.outputs.python }}"
+          python -m pip install -e . --verbose
+
+      - name: Run PennyLane-Lightning-GPU unit tests (MPICH backend)
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /opt/modules/
+          module load mpich
+          /opt/mpi/mpich/bin/mpirun -np 2 python -m pytest ./mpitests/ 
+          python -m pytest ./tests/ 
+
+      - name: Cleanup
+        if: always()
+        run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
+          rm -rf *
+          rm -rf .git
+          rm -rf .gitignore
+          rm -rf .github
+          pip cache purge