Post on demand benchmark results as comment (big refactor)

.github/workflows/benchmarks_report.yml

@@ -0,0 +1,83 @@
+# Post any reports generated by benchmarks_run.yml .
+# Separated for security:
+#  https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+
+name: benchmarks-report
+run-name: Report benchmark results
+
+on:
+  workflow_run:
+    workflows: [benchmarks-run]
+    types:
+      - completed
+
+jobs:
+  download:
+    runs-on: ubuntu-latest
+    outputs:
+      reports_exist: ${{ steps.unzip.outputs.reports_exist }}
+    steps:
+      - name: Download artifact
+        id: download-artifact
+        # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "benchmark_reports"
+            })[0];
+            if (typeof matchArtifact != 'undefined') {
+              let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+              });
+              let fs = require('fs');
+              fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/benchmark_reports.zip`, Buffer.from(download.data));
+            };
+
+      - name: Unzip artifact
+        id: unzip
+        run: |
+          if test -f "benchmark_reports.zip"; then
+            reports_exist=1
+            unzip benchmark_reports.zip -d benchmark_reports
+          else
+            reports_exist=0
+          fi
+          echo "reports_exist=$reports_exist" >> "$GITHUB_OUTPUT"
+
+      - name: Store artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark_reports
+          path: benchmark_reports
+
+  post_reports:
+    runs-on: ubuntu-latest
+    needs: download
+    if: needs.download.outputs.reports_exist == 1
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: benchmark_reports
+          path: .github/workflows/benchmark_reports
+
+      - name: Set up Python
+        # benchmarks/bm_runner.py only needs builtins to run.
+        uses: actions/setup-python@v3
+
+      - name: Post reports
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: python benchmarks/bm_runner.py _gh_post

.github/workflows/benchmark.yml → .github/workflows/benchmarks_run.yml

@@ -1,6 +1,9 @@
-# Use ASV to check for performance regressions in the last 24 hours' commits.
+# Use ASV to check for performance regressions, either:
+#  - In the last 24 hours' commits.
+#  - Introduced by this pull request.
 
-name: benchmark-check
+name: benchmarks-run
+run-name: Run benchmarks
 
 on:
   schedule:
@@ -74,12 +77,17 @@ jobs:
 
       - name: Benchmark this pull request
         if: ${{ github.event.label.name == 'benchmark_this' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.number }}
         run: |
           git checkout ${{ github.head_ref }}
           python benchmarks/bm_runner.py branch origin/${{ github.base_ref }}
 
       - name: Run overnight benchmarks
         if: ${{ github.event_name != 'pull_request' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           first_commit=${{ inputs.first_commit }}
           if [ "$first_commit" == "" ]
@@ -92,57 +100,15 @@ jobs:
             python benchmarks/bm_runner.py overnight $first_commit
           fi
 
-      - name: Create issues for performance shifts
-        if: ${{ github.event_name != 'pull_request' }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          if [ -d benchmarks/.asv/performance-shifts ]
-          then
-            cd benchmarks/.asv/performance-shifts
-            for commit_file in *
-            do
-              commit="${commit_file%.*}"
-              pr_number=$(git log "$commit"^! --oneline | grep -o "#[0-9]*" | tail -1 | cut -c 2-)
-              author=$(gh pr view $pr_number --json author -q '.["author"]["login"]' --repo $GITHUB_REPOSITORY)
-              merger=$(gh pr view $pr_number --json mergedBy -q '.["mergedBy"]["login"]' --repo $GITHUB_REPOSITORY)
-              # Find a valid assignee from author/merger/nothing.
-              if curl -s https://api.github.com/users/$author | grep -q '"type": "User"'; then
-                assignee=$author
-              elif curl -s https://api.github.com/users/$merger | grep -q '"type": "User"'; then
-                assignee=$merger
-              else
-                assignee=""
-              fi
-              title="Performance Shift(s): \`$commit\`"
-              body="
-          Benchmark comparison has identified performance shifts at
-
-          * commit $commit (#$pr_number).
-
-          Please review the report below and \
-          take corrective/congratulatory action as appropriate \
-          :slightly_smiling_face:
-
-          <details>
-          <summary>Performance shift report</summary>
-
-          \`\`\`
-          $(cat $commit_file)
-          \`\`\`
-
-          </details>
-
-          Generated by GHA run [\`${{github.run_id}}\`](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})
-              "
-              gh issue create --title "$title" --body "$body" --assignee $assignee --label "Bot" --label "Type: Performance" --repo $GITHUB_REPOSITORY
-            done
-          fi
+      - name: Upload any benchmark reports
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark_reports
+          path: .github/workflows/benchmark_reports
 
       - name: Archive asv results
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: asv-report
-          path: |
-            benchmarks/.asv/results
+          name: asv-raw-results
+          path: benchmarks/.asv/results

.gitignore

@@ -32,6 +32,7 @@ pip-cache
 # asv data, environments, results
 .asv
 benchmarks/.data
+.github/workflows/benchmark_reports
 
 #Translations
 *.mo