Add LGPU-MPI cpp backend

.github/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### New features since last release
 
+* Integrate the distributed C++ backend of Lightning-GPU into the Lightning monorepo.
+  [(#514)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/514)
+
 * Integrate Lightning-GPU into the Lightning monorepo. The new backend is named `lightning_gpu` and includes all single-GPU features.
   [(#499)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/499)
 

.github/workflows/tests_gpu_cu11.yml

@@ -1,4 +1,4 @@
-name: Testing::Linux::x86_64 (Lightning-GPU)
+name: Testing::Linux::x86_64::LGPU
 on:
   pull_request:
   push:
@@ -143,7 +143,7 @@ jobs:
           if-no-files-found: error
 
   pythontestswithLGPU:
-    needs: [builddeps]
+    needs: [builddeps, cpptestswithLGPU_cu11]
     strategy:
       matrix:
         os: [ubuntu-22.04]

.github/workflows/tests_linux_x86_mpi.yml

@@ -0,0 +1,185 @@
+name: Tests::Linux::x86_64::LGPU::MPI
+on:
+  workflow_call:
+    inputs:
+      lightning-version:
+        type: string
+        required: true
+        description: The version of lightning to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
+      pennylane-version:
+        type: string
+        required: true
+        description: The version of PennyLane to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
+  release:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  COVERAGE_FLAGS: "--cov=pennylane_lightning --cov-report=term-missing --cov-report=xml:./coverage.xml --no-flaky-report -p no:warnings --tb=native"
+  GCC_VERSION: 11
+  OMP_NUM_THREADS: "2"
+  CI_CUDA_ARCH: 86
+
+concurrency:
+  group: gpu-test-mpi-${{ github.ref }}-${{ inputs.lightning-version }}-${{ inputs.pennylane-version }}
+  cancel-in-progress: true
+
+jobs:
+  cpp_tests:
+    if: contains(github.event.pull_request.labels.*.name, 'ci:use-multi-gpu-runner') || (inputs.lightning-version != '' && inputs.pennylane-version != '')
+    runs-on:
+      - self-hosted
+      - linux
+      - x64
+      - ubuntu-22.04
+      - multi-gpu
+    strategy:
+      max-parallel: 1
+      matrix:
+        mpilib: ["mpich", "openmpi"]
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-tags: true
+
+      - name: Switch to stable build of Lightning-GPU
+        if: inputs.lightning-version == 'stable'
+        run: |
+          git fetch tags --force
+          git checkout $(git tag | sort -V | tail -1)
+      - uses: actions/setup-python@v4
+        id: setup_python
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      # Since the self-hosted runner can be re-used. It is best to set up all package
+      # installations in a virtual environment that gets cleaned at the end of each workflow run
+      - name: Setup Python virtual environment
+        id: setup_venv
+        env:
+          VENV_NAME: ${{ github.workspace }}/venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
+        run: |
+          # Clear any pre-existing venvs
+          rm -rf venv_*
+
+          # Create new venv for this workflow_run
+          python --version
+          python -m venv ${{ env.VENV_NAME }}
+
+          # Add the venv to PATH for subsequent steps
+          echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
+
+          # Adding venv name as an output for subsequent steps to reference if needed
+          echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
+
+      - name: Display Python-Path
+        id: python_path
+        run: |
+          py_path=$(which python)
+          echo "Python Interpreter Path => $py_path"
+          echo "python=$py_path" >> $GITHUB_OUTPUT
+
+          pip_path=$(which python)
+          echo "PIP Path => $pip_path"
+          echo "pip=$pip_path" >> $GITHUB_OUTPUT
+
+      - name: Install required packages
+        run: |
+          python -m pip install ninja cmake custatevec-cu11
+
+      - name: Validate GPU version and installed compiler
+        run: |
+          source /etc/profile.d/modules.sh && module use /opt/modules && module load cuda/11.8
+          which -a nvcc
+          nvcc --version
+      - name: Validate Multi-GPU packages
+        run: |
+          source /etc/profile.d/modules.sh && module use /opt/modules/ && module load ${{ matrix.mpilib }}
+          echo 'Checking for ${{ matrix.mpilib }}'
+          which -a mpirun
+          mpirun --version
+          which -a mpicxx
+          mpicxx --version
+          module unload ${{ matrix.mpilib }}
+      - name: Install Latest PennyLane
+        if: inputs.pennylane-version == 'latest'
+        run: python -m pip install git+https://github.com/PennyLaneAI/pennylane.git@master
+
+      - name: Build and run unit tests
+        run: |
+          source /etc/profile.d/modules.sh && module use /opt/modules/ && module load ${{ matrix.mpilib }}
+          export CUQUANTUM_SDK=$(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum/lib')")
+          cmake . -BBuild \
+            -DPL_BACKEND=lightning_gpu \
+            -DENABLE_MPI=ON \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DENABLE_COVERAGE=ON \
+            -DBUILD_TESTS=ON \
+            -DCMAKE_CXX_COMPILER=mpicxx \
+            -DCMAKE_CUDA_COMPILER="/usr/local/cuda/bin/nvcc" \
+            -DCMAKE_CUDA_ARCHITECTURES="86" \
+            -DPython_EXECUTABLE:FILE="${{ steps.python_path.outputs.python }}" \
+            -G Ninja
+          cmake --build ./Build
+          cd ./Build
+          mkdir -p ./tests/results
+          for file in *runner ; do ./$file --order lex --reporter junit --out ./tests/results/report_$file.xml; done;
+          for file in *runner_mpi ; do /opt/mpi/${{ matrix.mpilib }}/bin/mpirun -np 2 ./$file --order lex --reporter junit --out ./tests/results/report_$file.xml; done;
+          lcov --directory . -b ../pennylane_lightning/src --capture --output-file coverage.info
+          lcov --remove coverage.info '/usr/*' --output-file coverage.info
+          mv coverage.info coverage-${{ github.job }}-lightning_gpu_${{ matrix.mpilib }}.info
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          if-no-files-found: error
+          name: ubuntu-tests-reports
+          path: ./Build/tests/results/
+
+      - name: Upload code coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          if-no-files-found: error
+          name: ubuntu-codecov-results-cpp
+          path: ./Build/coverage-${{ github.job }}-lightning_gpu_${{ matrix.mpilib }}.info
+
+      - name: Cleanup
+        if: always()
+        run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
+          rm -rf * .git .gitignore .github
+          pip cache purge
+
+  upload-to-codecov-linux-cpp:
+    needs: ["cpp_tests"]
+    name: Upload coverage data to codecov
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Download coverage reports
+        uses: actions/download-artifact@v3
+        with:
+          name: ubuntu-codecov-results-cpp
+
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: true
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Cleanup
+        if: always()
+        run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
+          rm -rf * .git .gitignore .github
+          pip cache purge

Makefile

@@ -6,18 +6,24 @@ TESTRUNNER := -m pytest tests --tb=short
 
 PL_BACKEND?="$(if $(backend:-=),$(backend),lightning_qubit)"
 
-ifdef verbose
-    VERBOSE := --verbose
-else
-    VERBOSE :=
-endif
-
 ifdef check
     CHECK := --check
 else
     CHECK :=
 endif
 
+ifdef build_options
+    OPTIONS := $(build_options)
+else
+    OPTIONS :=
+endif
+
+ifdef verbose
+    VERBOSE := --verbose
+else
+    VERBOSE :=
+endif
+
 .PHONY: help
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -56,16 +62,7 @@ clean:
 	rm -rf .coverage coverage_html_report/
 	rm -rf pennylane_lightning/*_ops*
 
-.PHONY : test-builtin test-suite test-python coverage coverage-cpp test-cpp test-cpp-no-omp test-cpp-blas test-cpp-kokkos
-test-builtin:
-	PL_DEVICE=$(if $(device:-=),$(device),lightning.qubit) $(PYTHON) -I $(TESTRUNNER)
-
-test-suite:
-	pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --skip-ops --shots=20000
-	pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --shots=None --skip-ops
-
-test-python: test-builtin test-suite
-
+.PHONY : test-builtin test-suite test-python test-cpp coverage coverage-cpp 
 coverage:
 	@echo "Generating coverage report for $(if $(device:-=),$(device),lightning.qubit) device:"
 	$(PYTHON) $(TESTRUNNER) $(COVERAGE)
@@ -75,20 +72,29 @@ coverage:
 coverage-cpp:
 	@echo "Generating cpp coverage report in BuildCov/out for $(PL_BACKEND) backend"
 	rm -rf ./BuildCov
-	cmake -BBuildCov -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_COVERAGE=ON -DPL_BACKEND=$(PL_BACKEND)
+	cmake -BBuildCov -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_COVERAGE=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
 	cmake --build ./BuildCov
 	cd ./BuildCov; for file in *runner ; do ./$file; done; \
 	lcov --directory . -b ../pennylane_lightning/core/src --capture --output-file coverage.info; \
 	genhtml coverage.info --output-directory out
 
 build:
 	rm -rf ./Build
-	cmake -BBuild -G Ninja -DENABLE_BLAS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
+	cmake -BBuild -G Ninja -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
 	cmake --build ./Build $(VERBOSE)
 
+test-builtin:
+	PL_DEVICE=$(if $(device:-=),$(device),lightning.qubit) $(PYTHON) -I $(TESTRUNNER)
+
+test-suite:
+	pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --skip-ops --shots=20000
+	pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --shots=None --skip-ops
+
+test-python: test-builtin test-suite
+
 test-cpp:
 	rm -rf ./BuildTests
-	cmake -BBuildTests -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_OPENMP=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
+	cmake -BBuildTests -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
 ifdef target
 	cmake --build ./BuildTests $(VERBOSE) --target $(target)
 	OMP_PROC_BIND=false ./BuildTests/$(target)
@@ -97,12 +103,6 @@ else
 	OMP_PROC_BIND=false cmake --build ./BuildTests $(VERBOSE) --target test
 endif
 
-test-cpp-blas:
-	rm -rf ./BuildTests
-	cmake -BBuildTests -G Ninja -DBUILD_TESTS=ON -DENABLE_BLAS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
-	cmake --build ./BuildTests $(VERBOSE)
-	cmake --build ./BuildTests $(VERBOSE) --target test
-
 .PHONY: format format-cpp
 format: format-cpp format-python
 
@@ -115,7 +115,7 @@ format-python:
 .PHONY: check-tidy
 check-tidy:
 	rm -rf ./BuildTidy
-	cmake -BBuildTidy -DENABLE_CLANG_TIDY=ON -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
+	cmake -BBuildTidy -DENABLE_CLANG_TIDY=ON -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
 ifdef target
 	cmake --build ./BuildTidy $(VERBOSE) --target $(target)
 else

cmake/process_options.cmake

@@ -78,9 +78,6 @@ if(ENABLE_NATIVE)
     target_compile_options(lightning_compile_options INTERFACE -march=native)
 endif()
 
-if(ENABLE_MPI)
-    set(ENABLE_OPENMP OFF)
-endif()
 
 if(PLLGPU_DISABLE_CUDA_SAFETY)
     target_compile_options(lightning_compile_options INTERFACE $<$<COMPILE_LANGUAGE:CXX>:-DCUDA_UNSAFE>)

pennylane_lightning/core/_version.py

@@ -16,4 +16,4 @@
    Version number (major.minor.patch[-label])
 """
 
-__version__ = "0.33.0-dev22"
+__version__ = "0.33.0-dev23"

pennylane_lightning/core/src/algorithms/CMakeLists.txt

@@ -4,6 +4,10 @@ project(lightning_algorithms LANGUAGES CXX)
 
 add_library(lightning_algorithms INTERFACE)
 
+if(ENABLE_MPI)
+    target_compile_options(lightning_compile_options INTERFACE "-D_ENABLE_MPI=1")
+endif()
+
 target_include_directories(lightning_algorithms INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(lightning_algorithms INTERFACE    lightning_compile_options
                                                         lightning_external_libs

pennylane_lightning/core/src/algorithms/tests/CMakeLists.txt

@@ -43,3 +43,29 @@ target_link_libraries(algorithms_test_runner PRIVATE algorithms_tests)
 catch_discover_tests(algorithms_test_runner)
 
 install(TARGETS algorithms_test_runner DESTINATION bin)
+
+if(ENABLE_MPI)
+    add_library(algorithms_tests_mpi INTERFACE)
+    target_link_libraries(algorithms_tests_mpi INTERFACE     Catch2::Catch2)
+    foreach(BACKEND ${PL_BACKEND})
+        target_link_libraries(algorithms_tests_mpi INTERFACE  ${BACKEND}
+                                                            "${BACKEND}_gates"
+                                                            "${BACKEND}_measurements"
+                                                            "${BACKEND}_algorithms"
+                                                            "${BACKEND}_observables"
+        )
+    endforeach()
+
+    ProcessTestOptions(algorithms_tests_mpi)
+
+    target_sources(algorithms_tests_mpi INTERFACE ./mpi/runner_algorithms_mpi.cpp)
+
+    ################################################################################
+    # Define targets
+    ################################################################################
+    set(TEST_SOURCES    ./mpi/Test_AdjointJacobianMPI.cpp)
+
+    add_executable(algorithms_test_runner_mpi ${TEST_SOURCES})
+    target_link_libraries(algorithms_test_runner_mpi PRIVATE algorithms_tests_mpi)
+    catch_discover_tests(algorithms_test_runner_mpi)
+endif()