[pull] main from prisma:main #291
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "QE: WASM benchmarks" | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - ".github/**" | |
| - "!.github/workflows/wasm-benchmarks.yml" | |
| - ".buildkite/**" | |
| - "*.md" | |
| - "LICENSE" | |
| - "CODEOWNERS" | |
| - "renovate.json" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmarks: | |
| runs-on: ubuntu-latest | |
| env: # Set environment variables for the whole job | |
| PROFILE: release | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| # using head ref rather than merge branch to get original commit message | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - uses: ./.github/workflows/include/rust-wasm-setup | |
| - name: "Setup Node.js" | |
| uses: actions/setup-node@v4 | |
| - name: "Setup pnpm" | |
| uses: pnpm/[email protected] | |
| with: | |
| version: 8 | |
| - name: "Login to Docker Hub" | |
| uses: docker/login-action@v3 | |
| continue-on-error: true | |
| env: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: "${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}" | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Extract Branch Name | |
| run: | | |
| echo "Extracting branch name from: $(git show -s --format=%s)" | |
| branch="$(git show -s --format=%s | grep -o "DRIVER_ADAPTERS_BRANCH=[^ ]*" | cut -f2 -d=)" | |
| echo "branch=$branch" | |
| if [ -n "$branch" ]; then | |
| echo "Using $branch branch of driver adapters" | |
| echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV" | |
| fi | |
| - name: Setup benchmark | |
| run: make setup-pg-bench | |
| - name: Run benchmarks | |
| id: bench | |
| if: false | |
| run: | | |
| make run-bench | tee results.txt | |
| # Extract the values from the benchmark output | |
| regressed_values=$(grep "slower than Web Assembly: Latest" results.txt | cut -f1 -d'x') | |
| improved_values=$(grep "faster than Web Assembly: Latest" results.txt | cut -f1 -d'x') | |
| # Initialize sum variable and count | |
| total_sum=0 | |
| total_count=0 | |
| # Add the inverted regressed values to the sum | |
| for value in $regressed_values; do | |
| inverted=$(echo "scale=4; 1/$value" | bc) | |
| echo "Regressed value: $inverted" | |
| total_sum=$(echo "$total_sum + $inverted" | bc) | |
| total_count=$((total_count + 1)) | |
| done | |
| # Add the improved values to the sum | |
| for value in $improved_values; do | |
| echo "Improved value: $value" | |
| total_sum=$(echo "$total_sum + $value" | bc) | |
| total_count=$((total_count + 1)) | |
| done | |
| if [ $total_count -eq 0 ]; then | |
| echo "💥 something was wrong running the benchmarks" | |
| exit 1 | |
| fi | |
| mean=$(echo "scale=4; $total_sum / $total_count" | bc) | |
| echo "Extracted $total_count values from the benchmark output" | |
| echo "Total sum: $total_sum" | |
| echo "Total count: $total_count" | |
| echo "Mean: $mean" | |
| # Report improvement or worsening. Fails if >= 1.5% worsening. | |
| if (( $(echo "$mean < 0.985" | bc -l) )); then | |
| percent=$(echo "scale=4; ((1 / $mean) - 1) * 100" | bc) | |
| summary="❌ WASM query-engine performance will worsen by $(printf %.2f "$percent")%" | |
| status=failed | |
| elif (( $(echo "$mean > 1.015" | bc -l) )); then | |
| percent=$(echo "scale=4; ($mean - 1) * 100" | bc) | |
| summary="🚀 WASM query-engine performance will improve by $(printf %.2f "$percent")%" | |
| status=passed | |
| else | |
| delta=$(echo "scale=3; (1 / $mean)" | bc) | |
| summary="✅ WASM query-engine performance won't change substantially ($(printf %.3f "$delta")x)" | |
| status=passed | |
| fi | |
| echo "summary=$summary" >> "$GITHUB_OUTPUT" | |
| echo "status=$status" >> "$GITHUB_OUTPUT" | |
| # Save the output to a file so we can use it in the comment | |
| { | |
| echo 'bench_output<<EOF' | |
| cat results.txt | |
| echo EOF | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Find past report comment | |
| if: false | |
| uses: peter-evans/find-comment@v3 | |
| id: findReportComment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: "<!-- wasm-engine-perf -->" | |
| - name: Create or update report | |
| uses: peter-evans/create-or-update-comment@v4 | |
| # Only run on branches from our repository | |
| # It avoids an expected failure on forks | |
| # if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} | |
| if: false | |
| with: | |
| comment-id: ${{ steps.findReportComment.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body: | | |
| <!-- wasm-engine-perf --> | |
| #### ${{ steps.bench.outputs.summary }} | |
| <details> | |
| <summary>Full benchmark report</summary> | |
| ``` | |
| ${{ steps.bench.outputs.bench_output }} | |
| ``` | |
| </details> | |
| After changes in ${{ github.event.pull_request.head.sha }} | |
| edit-mode: replace | |
| - name: Fail workflow if regression detected | |
| # if: steps.bench.outputs.status == 'failed' | |
| if: false | |
| run: | | |
| echo "Workflow failed due to benchmark regression." | |
| exit 1 |