-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[mpact][benchmark] set up regression benchmark for each commit with g…
…raphs (#58)
- Loading branch information
1 parent
e7bd7ec
commit aa8d896
Showing
2 changed files
with
146 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
name: Regression benchmark | ||
|
||
on: | ||
push: | ||
branches: [ "main" ] | ||
|
||
permissions: | ||
contents: write | ||
deployments: write | ||
pull-requests: write | ||
repository-projects: write | ||
|
||
jobs: | ||
benchmark: | ||
name: Performance regression check | ||
runs-on: ubuntu-latest | ||
env: | ||
CACHE_DIR: ${{ github.workspace }}/.ccache | ||
PYTHONPATH: ${{ github.workspace }}/build/tools/mpact/python_packages/mpact | ||
steps: | ||
- uses: actions/checkout@v4 | ||
with: | ||
submodules: recursive | ||
token: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
- name: Setup Python Version | ||
uses: actions/setup-python@v5 | ||
with: | ||
python-version: 3.11 # Install the python version needed | ||
|
||
- name: Set up ccache | ||
uses: hendrikmuhs/[email protected] | ||
|
||
- name: Install requirements | ||
run: | | ||
export CCACHE_DIR=${{ env.CACHE_DIR }} | ||
python -m pip install --upgrade pip | ||
python -m pip install --upgrade pip | ||
python -m pip install pytest pytest-benchmark | ||
python -m pip install -r externals/torch-mlir/requirements.txt | ||
python -m pip install -r externals/torch-mlir/torchvision-requirements.txt | ||
- name: Create build directory | ||
run: mkdir build | ||
|
||
- name: Configure CMake | ||
run: > | ||
cmake -GNinja -Bbuild | ||
-DCMAKE_BUILD_TYPE=Release | ||
-DLLVM_ENABLE_PROJECTS=mlir | ||
-DLLVM_ENABLE_ASSERTIONS=ON | ||
-DLLVM_EXTERNAL_PROJECTS="torch-mlir;mpact" | ||
-DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="${PWD}/externals/torch-mlir" | ||
-DLLVM_EXTERNAL_MPACT_SOURCE_DIR="${PWD}" | ||
-DLLVM_TARGETS_TO_BUILD=host | ||
-DMLIR_ENABLE_BINDINGS_PYTHON=ON | ||
-DCMAKE_C_COMPILER_LAUNCHER=ccache | ||
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache | ||
-DCMAKE_C_COMPILER=clang | ||
-DCMAKE_CXX_COMPILER=clang++ | ||
"externals/torch-mlir/externals/llvm-project/llvm" | ||
- name: Build | ||
run: cmake --build build --target build-benchmark-mpact | ||
|
||
- name: Run benchmark | ||
run: pytest benchmark/python/benchmarks/regression_benchmark.py --benchmark-json output.json | ||
|
||
- name: Store benchmark result | ||
uses: benchmark-action/github-action-benchmark@v1 | ||
with: | ||
tool: 'pytest' | ||
output-file-path: output.json | ||
fail-on-alert: true | ||
# GitHub API token to make a commit comment | ||
github-token: ${{ secrets.GITHUB_TOKEN }} | ||
# Enable alert commit comment | ||
comment-on-alert: true | ||
# Mention @yinying-lisa-li in the commit comment | ||
alert-comment-cc-users: '@yinying-lisa-li' | ||
# Push and deploy GitHub pages branch automatically | ||
auto-push: true | ||
alert-threshold: 120% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
import pytest | ||
from mpact.models.kernels import * | ||
from mpact_benchmark.utils.tensor_generator import generate_tensor | ||
|
||
SHAPE = (1024, 1024) | ||
SPARSITY = 0.8 | ||
|
||
dense_tensor1 = generate_tensor(0, SHAPE, SPARSITY) | ||
dense_tensor2 = generate_tensor(1, SHAPE, SPARSITY) | ||
dense_tensor3 = generate_tensor(2, SHAPE, SPARSITY) | ||
dense_vector = generate_tensor(1, (SHAPE[0],), SPARSITY) | ||
|
||
sparse_tensor1 = dense_tensor1.to_sparse_csr() | ||
sparse_tensor2 = dense_tensor2.to_sparse_csr() | ||
sparse_tensor3 = dense_tensor3.to_sparse_csr() | ||
|
||
|
||
def test_mv_dense(benchmark): | ||
benchmark(MVNet(), dense_tensor1, dense_vector) | ||
|
||
|
||
def test_mm_dense(benchmark): | ||
benchmark(MMNet(), dense_tensor1, dense_tensor2) | ||
|
||
|
||
def test_add_dense(benchmark): | ||
benchmark(AddNet(), dense_tensor1, dense_tensor2) | ||
|
||
|
||
def test_mul_dense(benchmark): | ||
benchmark(MulNet(), dense_tensor1, dense_tensor2) | ||
|
||
|
||
def test_nop_dense(benchmark): | ||
benchmark(SelfNet(), dense_tensor1) | ||
|
||
|
||
def test_sddmm_dense(benchmark): | ||
benchmark(SDDMMNet(), dense_tensor1, dense_tensor2, dense_tensor3) | ||
|
||
|
||
def test_mv_sparse(benchmark): | ||
benchmark(MVNet(), sparse_tensor1, dense_vector) | ||
|
||
|
||
def test_mm_sparse(benchmark): | ||
benchmark(MMNet(), sparse_tensor1, sparse_tensor2) | ||
|
||
|
||
def test_add_sparse(benchmark): | ||
benchmark(AddNet(), sparse_tensor1, sparse_tensor2) | ||
|
||
|
||
def test_mul_sparse(benchmark): | ||
benchmark(MulNet(), sparse_tensor1, sparse_tensor2) | ||
|
||
|
||
def test_nop_sparse(benchmark): | ||
benchmark(SelfNet(), sparse_tensor1) | ||
|
||
|
||
def test_sddmm_sparse(benchmark): | ||
benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3) |