Refactor CI jobs for adding required jobs in Actions. (#2344)

* Add required jobs for Unit test. --------- Signed-off-by: Jay Zhang <[email protected]>
onnx · Jun 26, 2024 · 0004163 · 0004163
1 parent 1528091
commit 0004163
Show file tree

Hide file tree

Showing 12 changed files with 675 additions and 547 deletions.
diff --git a/.github/actions/keras_application_test/action.yml b/.github/actions/keras_application_test/action.yml
@@ -0,0 +1,75 @@
+name: Keras2onnx Application Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+          python -m pip install --upgrade pip
+          pip install onnxconverter-common
+          pip install onnx==${{ inputs.onnx_version }}
+          pip uninstall -y protobuf
+          pip install "protobuf~=3.20"
+          pip install h5py==3.7.0
+          pip install parameterized
+          pip install timeout-decorator
+          pip install coloredlogs flatbuffers
+          pip install tensorflow==${{ inputs.tf_version }}
+          pip install onnxruntime==${{ inputs.ort_version }}
+          pip install Pillow==8.2.0
+          pip install opencv-python
+          pip install tqdm
+          pip install keras-segmentation==0.2.0
+          git clone https://github.com/matterport/Mask_RCNN
+          cd Mask_RCNN
+          pip install -r requirements.txt
+          python setup.py install
+          cd ..
+          pip install matplotlib
+          git clone https://github.com/qqwweee/keras-yolo3
+          pip install keras-resnet
+          pip install git+https://www.github.com/keras-team/keras-contrib.git
+          pip install keras-tcn==2.8.3
+          pip install git+https://github.com/qubvel/efficientnet
+          pip install keras-self-attention
+          pip install pytest pytest-cov pytest-runner
+
+          if [[ ${{ inputs.tf_version }} == 1.* ]]; then 
+              pip install keras==2.3.1
+              pip install transformers==4.2.0
+              pip uninstall -y h5py
+              pip install h5py==2.9.0
+              pip install numpy==1.19.0
+          else 
+              pip install transformers
+              pip install "numpy<2"
+          fi
+
+          pip install -e .
+
+          echo "----- List all of depdencies:"
+          pip freeze --all
+
+    - name: Run keras_application_test (${{ runner.os }})
+      shell: bash
+      run: |
+          python -c "import onnxruntime"
+          pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
+          cd tests/keras2onnx_applications/nightly_build
+          python run_all_v2.py
diff --git a/.github/actions/keras_unit_test/action.yml b/.github/actions/keras_unit_test/action.yml
@@ -0,0 +1,53 @@
+name: Keras2onnx Unit Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+          python -m pip install --upgrade pip
+          pip install onnxconverter-common
+          pip install onnx==${{ inputs.onnx_version }}
+          pip install h5py==3.7.0
+          pip install parameterized
+          pip install timeout-decorator
+          pip install coloredlogs flatbuffers
+          pip install tensorflow==${{ inputs.tf_version }}
+          pip install pytest pytest-cov pytest-runner
+          pip install onnxruntime==${{ inputs.ort_version }}
+          pip uninstall -y protobuf
+          pip install "protobuf~=3.20"
+          if [[ ${{ inputs.tf_version }} == 1.* ]]; then 
+              pip install numpy==1.19.0
+          else 
+              pip install "numpy<2"
+          fi
+
+          pip install -e .
+
+          echo "----- List all of depdencies:"
+          pip freeze --all
+
+    - name: Run keras_unit_test (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+          python -c "import onnxruntime"
+          python -c "import onnxconverter_common"
+          pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
diff --git a/.github/actions/prepare_env/action.yml b/.github/actions/prepare_env/action.yml
diff --git a/.github/actions/pretrained_model_test/action.yml b/.github/actions/pretrained_model_test/action.yml
@@ -0,0 +1,46 @@
+name: Pretrained Model Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run pretrained_model_test
+      shell: bash
+      run: |
+        # TODO: fix unity model path
+        # python tests/run_pretrained_models.py --backend $CI_ONNX_BACKEND --opset $CI_ONNX_OPSET --config tests/unity.yaml || status=$?
+        python tests/run_pretrained_models.py --backend onnxruntime --opset ${{ inputs.opset_version }} --skip_tf_tests False --skip_tflite_tests ${{ inputs.skip_tflite }} --skip_tfjs_tests True --config tests/run_pretrained_models.yaml || status=$?
+        ls
diff --git a/.github/actions/unit_test/action.yml b/.github/actions/unit_test/action.yml
@@ -0,0 +1,62 @@
+name: Unit Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run unit_test (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        export TF2ONNX_TEST_BACKEND=onnxruntime
+        export TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        export TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        export TF2ONNX_SKIP_TFJS_TESTS=True
+        export TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
+
+    - name: Run unit_test (Windows)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        set TF2ONNX_TEST_BACKEND=onnxruntime
+        set TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        set TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        set TF2ONNX_SKIP_TFJS_TESTS=True
+        set TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls