chore: improved benchmark reporting and gating

.github/workflows/build.yml

@@ -25,6 +25,7 @@ env:
   CARGO_TERM_COLOR: always
   TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
   TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+  IS_SAME_REPO_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
 
 jobs:
   build:
@@ -182,18 +183,20 @@ jobs:
         run: pnpm install --ignore-scripts --frozen-lockfile --filter hangar --filter examples-valid --filter examples-invalid
 
       - name: Run E2E Benchmarks
-        run: pnpm bench
+        env:
+          BENCH_FAIL_THRESHOLD_PERCENT: "50"
+          BENCH_COMPARE_PREVIOUS: "${{ github.event_name == 'pull_request' && 'main' || '' }}"
+          GITHUB_TOKEN: ${{ env.IS_SAME_REPO_PR == 'true' && secrets.PROJEN_GITHUB_TOKEN || secrets.GITHUB_TOKEN  }}
+          BENCH_PR: "${{ env.IS_SAME_REPO_PR == 'true' && github.event.pull_request.number || '' }}"
         working-directory: tools/hangar
+        run: pnpm bench
 
       - name: Upload Report JSON
         uses: actions/upload-artifact@v3
         with:
           name: benchmarks
           path: tools/hangar/results/report.json
 
-      - name: Create Markdown report
-        run: scripts/benchmark_json_to_table.mjs
-
   e2e-test:
     name: "E2E / ${{ matrix.runner }} + Node${{ matrix.node }} [${{ matrix.shard }}]"
     runs-on: "${{ matrix.runner }}-latest"
@@ -314,8 +317,8 @@ jobs:
         with:
           message: |
             Console preview environment is available at https://${{env.APP_NAME}}.fly.dev :rocket:
-
-            *Updated (UTC): ${{steps.deploy-fly.outputs.deploytime }}*
+            
+            ###### Last Updated (UTC) ${{steps.deploy-fly.outputs.deploytime }}
           comment_tag: Console preview environment
           GITHUB_TOKEN: ${{secrets.PROJEN_GITHUB_TOKEN}}
 

apps/wing/package.json

@@ -26,7 +26,7 @@
     "copy-root-assets": "cp ../../logo/demo.gif ./logo/demo.gif",
     "watch": "tsc --watch",
     "test": "vitest run",
-    "dev": "pnpm build && ./bin/wing",
+    "dev": "./bin/wing",
     "package": "bump-pack -b"
   },
   "dependencies": {

docs/contributing/01-start-here/05-development.md

@@ -120,15 +120,17 @@ If you wish to install it manually, you may do so by running `scripts/setup_wasi
 
 ## 🧪 How do I run tests?
 
-End-to-end tests are hosted under `./tools/hangar`. To get started, first ensure you can [build
+End-to-end tests are hosted under `tools/hangar`. To get started, first ensure you can [build
 wing](#-how-do-i-build-wing).
 
 To run the tests (and update snapshots), run the following command from anywhere in the monorepo:
 
 ```sh
-pnpm turbo test --filter=hangar
+pnpm turbo wing:e2e
 ```
 
+(This is a helpful shortcut for `pnpm turbo test --filter=hangar`)
+
 ### Test Meta-Comments
 
 In your wing files in `examples/tests/valid`, you can add a specially formatted comment to add additional information for hangar.
@@ -153,24 +155,12 @@ This is useful if, for example, the test requires docker. In our CI only linux s
 Benchmark files are located in `examples/tests/valid/benchmarks`. To run the benchmarks, run the following command from anywhere in the monorepo:
 
 ```sh
-pnpm turbo bench
+pnpm turbo wing:bench
 ```
 
-Benchmark files should ideally have a meta-comment with the `cases` key. For example:
-
-```ts
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 900
-  - target: tf-aws
-    maxMeanTime: 1000
-\*/
-```
+(This is a helpful shortcut for `pnpm turbo bench --filter=hangar`)
 
-Given each of these cases, the current purpose is to provide a maxMeanTime (milliseconds) per compilation target.
-If the average time for compiling to this target takes longer than the maxMeanTime, the test will fail.
-Note: In CI, tests likely run much slower than on your local machine, so you may need to observe the CI results to determine the correct maxMeanTime.
+In CI, if these benchmarks regress too far from the current `main` branch, the build will fail.
 
 ## How do I work only on the compiler?
 

examples/tests/valid/benchmarks/empty.test.w

@@ -1,7 +0,0 @@
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 2900
-  - target: tf-aws
-    maxMeanTime: 3000
-\*/

examples/tests/valid/benchmarks/functions_1.test.w

@@ -1,11 +1,3 @@
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 3000
-  - target: tf-aws
-    maxMeanTime: 8000
-\*/
-
 bring cloud;
 
 new cloud.Function(inflight () => { assert(1 == 1); }) as "Function1";

examples/tests/valid/benchmarks/functions_10.test.w

@@ -1,11 +1,3 @@
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 3000
-  - target: tf-aws
-    maxMeanTime: 22000
-\*/
-
 bring cloud;
 
 new cloud.Function(inflight () => { assert(1 == 1); }) as "Function1";

examples/tests/valid/benchmarks/hello_world.test.w

@@ -1,11 +1,3 @@
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 3000
-  - target: tf-aws
-    maxMeanTime: 8000
-\*/
-
 bring cloud;
 
 let bucket = new cloud.Bucket();

examples/tests/valid/benchmarks/jsii_big.test.w

@@ -1,9 +1 @@
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 11000
-  - target: tf-aws
-    maxMeanTime: 11000
-\*/
-
 bring "@cdktf/provider-aws" as aws;

examples/tests/valid/benchmarks/jsii_small.test.w

@@ -1,9 +1 @@
-/*\
-cases:
-  - target: sim
-    maxMeanTime: 2900
-  - target: tf-aws
-    maxMeanTime: 3000
-\*/
-
 bring "jsii-code-samples" as stuff;