benchmark-check #1535
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Use ASV to check for performance regressions in the last 24 hours' commits. | |
| name: benchmark-check | |
| on: | |
| schedule: | |
| # Runs every day at 23:00. | |
| - cron: "0 23 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| first_commit: | |
| description: "Argument to be passed to the overnight benchmark script." | |
| required: false | |
| type: string | |
| pull_request: | |
| types: [labeled] | |
| jobs: | |
| benchmark: | |
| if: > | |
| github.repository == 'SciTools/iris' && | |
| (github.event_name != 'pull_request' || | |
| github.event.label.name == 'benchmark_this') | |
| runs-on: ubuntu-latest | |
| env: | |
| IRIS_TEST_DATA_LOC_PATH: benchmarks | |
| IRIS_TEST_DATA_PATH: benchmarks/iris-test-data | |
| IRIS_TEST_DATA_VERSION: "2.19" | |
| # Lets us manually bump the cache to rebuild | |
| ENV_CACHE_BUILD: "0" | |
| TEST_DATA_CACHE_BUILD: "2" | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install ASV & Nox | |
| run: | | |
| pip install asv nox | |
| - name: Cache environment directories | |
| id: cache-env-dir | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .nox | |
| benchmarks/.asv/env | |
| $CONDA/pkgs | |
| key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }} | |
| - name: Cache test data directory | |
| id: cache-test-data | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ${{ env.IRIS_TEST_DATA_PATH }} | |
| key: | |
| test-data-${{ env.IRIS_TEST_DATA_VERSION }}-${{ env.TEST_DATA_CACHE_BUILD }} | |
| - name: Fetch the test data | |
| if: steps.cache-test-data.outputs.cache-hit != 'true' | |
| run: | | |
| wget --quiet https://github.com/SciTools/iris-test-data/archive/v${IRIS_TEST_DATA_VERSION}.zip -O iris-test-data.zip | |
| unzip -q iris-test-data.zip | |
| mkdir --parents ${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_LOC_PATH} | |
| mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH} | |
| - name: Set test data var | |
| run: | | |
| echo "OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH}/test_data" >> $GITHUB_ENV | |
| - name: Benchmark this pull request | |
| if: ${{ github.event.label.name == 'benchmark_this' }} | |
| run: | | |
| git checkout ${{ github.head_ref }} | |
| python benchmarks/bm_runner.py branch origin/${{ github.base_ref }} | |
| - name: Run overnight benchmarks | |
| if: ${{ github.event_name != 'pull_request' }} | |
| run: | | |
| first_commit=${{ inputs.first_commit }} | |
| if [ "$first_commit" == "" ] | |
| then | |
| first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1) | |
| fi | |
| if [ "$first_commit" != "" ] | |
| then | |
| python benchmarks/bm_runner.py overnight $first_commit | |
| fi | |
| - name: Create issues for performance shifts | |
| if: ${{ github.event_name != 'pull_request' }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ -d benchmarks/.asv/performance-shifts ] | |
| then | |
| cd benchmarks/.asv/performance-shifts | |
| for commit_file in * | |
| do | |
| commit="${commit_file%.*}" | |
| pr_number=$(git log "$commit"^! --oneline | grep -o "#[0-9]*" | tail -1 | cut -c 2-) | |
| author=$(gh pr view $pr_number --json author -q '.["author"]["login"]' --repo $GITHUB_REPOSITORY) | |
| merger=$(gh pr view $pr_number --json mergedBy -q '.["mergedBy"]["login"]' --repo $GITHUB_REPOSITORY) | |
| # Find a valid assignee from author/merger/nothing. | |
| if curl -s https://api.github.com/users/$author | grep -q '"type": "User"'; then | |
| assignee=$author | |
| elif curl -s https://api.github.com/users/$merger | grep -q '"type": "User"'; then | |
| assignee=$merger | |
| else | |
| assignee="" | |
| fi | |
| title="Performance Shift(s): \`$commit\`" | |
| body=" | |
| Benchmark comparison has identified performance shifts at | |
| * commit $commit (#$pr_number). | |
| Please review the report below and \ | |
| take corrective/congratulatory action as appropriate \ | |
| :slightly_smiling_face: | |
| <details> | |
| <summary>Performance shift report</summary> | |
| \`\`\` | |
| $(cat $commit_file) | |
| \`\`\` | |
| </details> | |
| Generated by GHA run [\`${{github.run_id}}\`](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}) | |
| " | |
| gh issue create --title "$title" --body "$body" --assignee $assignee --label "Bot" --label "Type: Performance" --repo $GITHUB_REPOSITORY | |
| done | |
| fi | |
| - name: Archive asv results | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: asv-report | |
| path: | | |
| benchmarks/.asv/results |