[mpact][benchmark] set up regression benchmark for each commit with g…

…raphs (#58)

.github/workflows/regression-benchmark.yml

@@ -0,0 +1,83 @@
+name: Regression benchmark
+
+on:
+  push:
+    branches: [ "main" ]
+
+permissions:
+  contents: write
+  deployments: write
+  pull-requests: write
+  repository-projects: write
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    env:
+      CACHE_DIR: ${{ github.workspace }}/.ccache
+      PYTHONPATH: ${{ github.workspace }}/build/tools/mpact/python_packages/mpact    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          submodules: recursive
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Setup Python Version
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11 # Install the python version needed
+
+    - name: Set up ccache
+      uses: hendrikmuhs/[email protected]
+
+    - name: Install requirements
+      run: |
+           export CCACHE_DIR=${{ env.CACHE_DIR }}
+           python -m pip install --upgrade pip
+           python -m pip install --upgrade pip
+           python -m pip install pytest pytest-benchmark
+           python -m pip install -r externals/torch-mlir/requirements.txt
+           python -m pip install -r externals/torch-mlir/torchvision-requirements.txt
+   
+    - name: Create build directory
+      run: mkdir build
+
+    - name: Configure CMake
+      run: >
+        cmake -GNinja -Bbuild
+        -DCMAKE_BUILD_TYPE=Release
+        -DLLVM_ENABLE_PROJECTS=mlir
+        -DLLVM_ENABLE_ASSERTIONS=ON
+        -DLLVM_EXTERNAL_PROJECTS="torch-mlir;mpact"
+        -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="${PWD}/externals/torch-mlir"
+        -DLLVM_EXTERNAL_MPACT_SOURCE_DIR="${PWD}"
+        -DLLVM_TARGETS_TO_BUILD=host
+        -DMLIR_ENABLE_BINDINGS_PYTHON=ON
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER=clang
+        -DCMAKE_CXX_COMPILER=clang++
+        "externals/torch-mlir/externals/llvm-project/llvm"
+
+    - name: Build
+      run: cmake --build build --target build-benchmark-mpact
+
+    - name: Run benchmark
+      run: pytest benchmark/python/benchmarks/regression_benchmark.py --benchmark-json output.json
+
+    - name: Store benchmark result
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        tool: 'pytest'
+        output-file-path: output.json
+        fail-on-alert: true
+        # GitHub API token to make a commit comment
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        # Enable alert commit comment
+        comment-on-alert: true
+        # Mention @yinying-lisa-li in the commit comment
+        alert-comment-cc-users: '@yinying-lisa-li'
+        # Push and deploy GitHub pages branch automatically
+        auto-push: true
+        alert-threshold: 120%

benchmark/python/benchmarks/regression_benchmark.py

@@ -0,0 +1,63 @@
+import pytest
+from mpact.models.kernels import *
+from mpact_benchmark.utils.tensor_generator import generate_tensor
+
+SHAPE = (1024, 1024)
+SPARSITY = 0.8
+
+dense_tensor1 = generate_tensor(0, SHAPE, SPARSITY)
+dense_tensor2 = generate_tensor(1, SHAPE, SPARSITY)
+dense_tensor3 = generate_tensor(2, SHAPE, SPARSITY)
+dense_vector = generate_tensor(1, (SHAPE[0],), SPARSITY)
+
+sparse_tensor1 = dense_tensor1.to_sparse_csr()
+sparse_tensor2 = dense_tensor2.to_sparse_csr()
+sparse_tensor3 = dense_tensor3.to_sparse_csr()
+
+
+def test_mv_dense(benchmark):
+    benchmark(MVNet(), dense_tensor1, dense_vector)
+
+
+def test_mm_dense(benchmark):
+    benchmark(MMNet(), dense_tensor1, dense_tensor2)
+
+
+def test_add_dense(benchmark):
+    benchmark(AddNet(), dense_tensor1, dense_tensor2)
+
+
+def test_mul_dense(benchmark):
+    benchmark(MulNet(), dense_tensor1, dense_tensor2)
+
+
+def test_nop_dense(benchmark):
+    benchmark(SelfNet(), dense_tensor1)
+
+
+def test_sddmm_dense(benchmark):
+    benchmark(SDDMMNet(), dense_tensor1, dense_tensor2, dense_tensor3)
+
+
+def test_mv_sparse(benchmark):
+    benchmark(MVNet(), sparse_tensor1, dense_vector)
+
+
+def test_mm_sparse(benchmark):
+    benchmark(MMNet(), sparse_tensor1, sparse_tensor2)
+
+
+def test_add_sparse(benchmark):
+    benchmark(AddNet(), sparse_tensor1, sparse_tensor2)
+
+
+def test_mul_sparse(benchmark):
+    benchmark(MulNet(), sparse_tensor1, sparse_tensor2)
+
+
+def test_nop_sparse(benchmark):
+    benchmark(SelfNet(), sparse_tensor1)
+
+
+def test_sddmm_sparse(benchmark):
+    benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3)