[feature][refactor] Optimum-Benchmark API (#118)

.github/workflows/check_code_quality.yaml → .github/workflows/check_quality.yaml

@@ -1,4 +1,4 @@
-name: Quality Code Checks
+name: Quality Check
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_code_quality_checks:
+  run_quality_checks:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -29,6 +29,4 @@ jobs:
           pip install -e .[quality]
 
       - name: Check style
-        run: |
-          black --check .
-          ruff check .
+        run: make quality

.github/workflows/test_api_cpu.yaml

@@ -0,0 +1,32 @@
+name: API CPU Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run_api_cpu_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -e .[testing,timm,diffusers]
+
+      - name: Run tests
+        run: pytest -k "api and cpu"

.github/workflows/test_api_cuda.yaml

@@ -0,0 +1,53 @@
+name: API CUDA Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_image_and_run_api_cuda_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          [
+            { torch_cuda: cu121, cuda_version: 12.1.1 },
+            { torch_cuda: cu118, cuda_version: 11.8.0 },
+          ]
+
+    runs-on: hf-dgx-01
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build image
+        run: docker build
+          --file docker/cuda.dockerfile
+          --build-arg USER_ID=$(id -u)
+          --build-arg GROUP_ID=$(id -g)
+          --build-arg TORCH_CUDA=${{ matrix.image.torch_cuda }}
+          --build-arg CUDA_VERSION=${{ matrix.image.cuda_version }}
+          --tag opt-bench-cuda:${{ matrix.image.cuda_version }}
+          .
+
+      - name: Run tests
+        run: docker run
+          --rm
+          --net host
+          --pid host
+          --shm-size 64G
+          --env USE_CUDA="1"
+          --volume $HOME/.cache/huggingface:/home/user/.cache/huggingface
+          --volume $(pwd):/workspace/optimum-benchmark
+          --workdir /workspace/optimum-benchmark
+          --gpus '"device=0,1"'
+          --entrypoint /bin/bash
+          opt-bench-cuda:${{ matrix.image.cuda_version }}
+          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'api and cuda' -x"

.github/workflows/test_api_misc.yaml

@@ -0,0 +1,32 @@
+name: API Misc Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run_api_misc_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+
+      - name: Install requirements
+        run: |
+          pip install --upgrade pip
+          pip install -e .[testing,timm,diffusers]
+
+      - name: Run tests
+        run: pytest -k "api and not (cpu or cuda or rocm or tensorrt)"

.github/workflows/test_api_rocm.yaml

@@ -0,0 +1,56 @@
+name: API ROCm Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_image_and_run_api_rocm_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          [
+            { torch_rocm: rocm5.6, torch_pre_release: 0, rocm_version: 5.6.1 },
+            { torch_rocm: rocm5.7, torch_pre_release: 1, rocm_version: 5.7 },
+          ]
+
+    runs-on: hf-amd-mi210-dev
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Build image
+        run: docker build
+          --file docker/rocm.dockerfile
+          --build-arg USER_ID=$(id -u)
+          --build-arg GROUP_ID=$(id -g)
+          --build-arg TORCH_ROCM=${{ matrix.image.torch_rocm }}
+          --build-arg TORCH_PRE_RELEASE=${{ matrix.image.torch_pre_release }}
+          --build-arg ROCM_VERSION=${{ matrix.image.rocm_version }}
+          --tag opt-bench-rocm:${{ matrix.image.rocm_version }}
+          .
+
+      - name: Run tests
+        run: docker run
+          --rm
+          --net host
+          --pid host
+          --shm-size 64G
+          --env USE_ROCM="1"
+          --volume $HOME/.cache/huggingface:/home/user/.cache/huggingface
+          --volume $(pwd):/workspace/optimum-benchmark
+          --workdir /workspace/optimum-benchmark
+          --device /dev/kfd
+          --device /dev/dri/renderD128
+          --device /dev/dri/renderD129
+          --entrypoint /bin/bash
+          opt-bench-rocm:${{ matrix.image.rocm_version }}
+          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'api and cuda' -x"

.github/workflows/test_cli_misc.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_misc_tests:
+  run_misc_cli_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -26,8 +26,7 @@ jobs:
       - name: Install requirements
         run: |
           pip install --upgrade pip
-          pip install -e .[test]
+          pip install -e .[testing]
 
       - name: Run tests
-        run: |
-          pytest -k "not (cpu or cuda or rocm or tensorrt)"
+        run: pytest -k "cli and not (cpu or cuda or rocm or tensorrt)"

.github/workflows/test_cpu_neural_compressor.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_tests:
+  run_cpu_neural_compressor_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -26,8 +26,7 @@ jobs:
       - name: Install Intel Neural Compressor CPU requirements
         run: |
           pip install --upgrade pip
-          pip install -e .[test,neural-compressor]
+          pip install -e .[testing,neural-compressor,diffusers,timm]
 
       - name: Run Intel Neural Compressor CPU tests
-        run: |
-          pytest -k "cpu and neural_compressor"
+        run: pytest -k "cli and cpu and neural_compressor"

.github/workflows/test_cpu_onnxruntime.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_tests:
+  run_cpu_onnxruntime_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -26,8 +26,8 @@ jobs:
       - name: Install requirements
         run: |
           pip install --upgrade pip
-          pip install -e .[test,onnxruntime,diffusers,timm]
+          pip install -e .[testing,onnxruntime,diffusers,timm]
 
       - name: Run tests
         run: |
-          pytest -k "cpu and onnxruntime"
+          pytest -k "cli and cpu and onnxruntime"

.github/workflows/test_cpu_openvino.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_tests:
+  run_cpu_openvino_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -26,8 +26,8 @@ jobs:
       - name: Install requirements
         run: |
           pip install --upgrade pip
-          pip install -e .[test,openvino,diffusers]
+          pip install -e .[testing,openvino,diffusers,timm]
 
       - name: Run tests
         run: |
-          pytest -k "cpu and openvino"
+          pytest -k "cli and cpu and openvino"

.github/workflows/test_cpu_pytorch.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_tests:
+  run_cpu_pytorch_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -26,8 +26,8 @@ jobs:
       - name: Install requirements
         run: |
           pip install --upgrade pip
-          pip install -e .[test,diffusers,timm]
+          pip install -e .[testing,diffusers,timm,peft]
 
       - name: Run tests
         run: |
-          pytest -k "cpu and pytorch"
+          pytest -k "cli and cpu and pytorch"

.github/workflows/test_cuda_onnxruntime_inference.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_gpu_tests:
+  build_image_and_run_cuda_onnxruntime_inference_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout
@@ -40,4 +40,4 @@ jobs:
           --workdir /workspace/optimum-benchmark
           --gpus '"device=0,1"'
           opt-bench-cuda:11.8.0
-          -c "pip install -e .[test,onnxruntime-gpu] && pytest -k 'cuda and onnxruntime and inference' -x"
+          -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and cuda and onnxruntime and inference' -x"

.github/workflows/test_cuda_pytorch.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_gpu_tests:
+  build_image_and_run_cuda_pytorch_tests:
     strategy:
       fail-fast: false
       matrix:
@@ -50,4 +50,4 @@ jobs:
           --gpus '"device=0,1"'
           --entrypoint /bin/bash
           opt-bench-cuda:${{ matrix.image.cuda_version }}
-          -c "pip install -e .[test,peft,deepspeed] && pytest -k 'cuda and pytorch' -x"
+          -c "pip install -e .[testing,diffusers,timm,deepspeed,peft] && pytest -k 'cli and cuda and pytorch' -x"

.github/workflows/test_cuda_onnxruntime_training.yaml → .github/workflows/test_cuda_torch_ort_training.yaml

@@ -1,4 +1,4 @@
-name: CUDA OnnxRuntime Training Tests
+name: CUDA Torch-ORT Training Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_gpu_tests:
+  build_image_and_run_cuda_torch_ort_training_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout
@@ -40,4 +40,4 @@ jobs:
           --workdir /workspace/optimum-benchmark
           --gpus '"device=0,1"'
           opt-bench-cuda:11.8.0
-          -c "pip install -e .[test,onnxruntime-training,peft] && python -m torch_ort.configure && pytest -k 'cuda and onnxruntime and training' -x"
+          -c "pip install -e .[testing,torch-ort,peft] && python -m torch_ort.configure && pytest -k 'cli and cuda and torch_ort and training' -x"

.github/workflows/test_rocm_onnxruntime_inference.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_gpu_tests:
+  build_image_and_run_rocm_onnxruntime_inference_tests:
     runs-on: hf-amd-mi210-dev
     steps:
       - name: Checkout
@@ -51,4 +51,4 @@ jobs:
           --device /dev/dri/renderD129
           --entrypoint /bin/bash
           opt-bench-rocm-ort:5.7
-          -c "pip install -e .[test,diffusers] && pytest -k 'rocm and onnxruntime and inference' -x"
+          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'cli and rocm and onnxruntime and inference' -x"

.github/workflows/test_rocm_pytorch.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_gpu_tests:
+  build_image_and_run_rocm_pytorch_tests:
     strategy:
       fail-fast: false
       matrix:
@@ -53,4 +53,4 @@ jobs:
           --device /dev/dri/renderD129
           --entrypoint /bin/bash
           opt-bench-rocm:${{ matrix.image.rocm_version }}
-          -c "pip install -e .[test,peft,deepspeed] && pytest -k 'cuda and pytorch' -x"
+          -c "pip install -e .[testing,diffusers,timm,deepspeed,peft] && pytest -k 'cli and cuda and pytorch' -x"

.github/workflows/test_tensorrt_llm.yaml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pull_image_and_run_gpu_tests:
+  pull_image_and_run_tensorrt_llm_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout
@@ -40,4 +40,4 @@ jobs:
           --gpus '"device=0,1"'
           --entrypoint /bin/bash
           opt-bench-tensorrt-llm:latest
-          -c "pip install -e .[test] && pytest -k 'tensorrt_llm' -x"
+          -c "pip install -e .[testing] && pytest -k 'cli and tensorrt_llm' -x"