chore: rerun baseline benchmark as part of workflow

.github/workflows/build.yml

@@ -182,10 +182,16 @@ jobs:
       - name: Install Dependencies
         run: pnpm install --ignore-scripts --frozen-lockfile --filter hangar --filter examples-valid --filter examples-invalid
 
-      - name: Run E2E Benchmarks
+      - name: Run Baseline Benchmarks
+        working-directory: tools/hangar
+        if: github.event_name == 'pull_request'
+        env:
+          HANGAR_WINGLANG_PACKAGE: "winglang@latest"
+        run: pnpm bench
+
+      - name: Run Benchmarks
         env:
-          BENCH_FAIL_THRESHOLD_PERCENT: "50"
-          BENCH_COMPARE_PREVIOUS: "${{ github.event_name == 'pull_request' && 'main' || '' }}"
+          BENCH_FAIL_THRESHOLD_PERCENT: "25"
           GITHUB_TOKEN: ${{ env.IS_SAME_REPO_PR == 'true' && secrets.PROJEN_GITHUB_TOKEN || secrets.GITHUB_TOKEN  }}
           BENCH_PR: "${{ env.IS_SAME_REPO_PR == 'true' && github.event.pull_request.number || '' }}"
         working-directory: tools/hangar

tools/hangar/src/benchmarking/cli.bench.ts

@@ -33,7 +33,7 @@ describe("compile", async () => {
         },
         {
           warmupIterations: 1,
-          iterations: 15,
+          iterations: 10,
           time: 0,
           setup: async (f) => {
             f.opts.beforeEach = async () => {

tools/hangar/src/benchmarking/compare.ts

@@ -6,10 +6,13 @@ import { createTable } from "./table_report";
 interface ProcessedBenchData {
   mean: number;
   moe: number;
+  sd: number;
   name: string;
 }
 
-function avgBenches(benchList: any[]): Record<string, ProcessedBenchData | undefined> {
+function avgBenches(
+  benchList: any[]
+): Record<string, ProcessedBenchData | undefined> {
   const benchValues: Record<string, ProcessedBenchData[]> = {};
   for (const bench of benchList) {
     for (const item of bench) {
@@ -20,6 +23,7 @@ function avgBenches(benchList: any[]): Record<string, ProcessedBenchData | undef
         name: item.name,
         mean: item["mean"],
         moe: item["moe"],
+        sd: item["sd"],
       };
       benchValues[item.name].push(data);
     }
@@ -30,10 +34,12 @@ function avgBenches(benchList: any[]): Record<string, ProcessedBenchData | undef
     const data = benchValues[key];
     const mean = data.reduce((acc, cur) => acc + cur.mean, 0) / data.length;
     const moe = data.reduce((acc, cur) => acc + cur.moe, 0) / data.length;
+    const sd = data.reduce((acc, cur) => acc + cur.sd, 0) / data.length;
     returnData[key] = {
       name: key,
       mean,
       moe,
+      sd,
     };
   }
 
@@ -92,15 +98,29 @@ export async function compareBenchmarks(
     differences[itemName].moeAfter = newData?.moe ?? NaN;
     differences[itemName].meanAfter = newData?.mean ?? NaN;
 
+    differences[itemName].maxSD = Math.max(
+      newData?.sd ?? NaN,
+      oldData?.sd ?? NaN
+    ) * 1.5;
+
     differences[itemName].meanDiff =
-      Math.round((differences[itemName].meanAfter - differences[itemName].meanBefore) * 100) / 100;
+      Math.round(
+        (differences[itemName].meanAfter - differences[itemName].meanBefore) *
+          100
+      ) / 100;
     differences[itemName].meanPercentDiff =
-      Math.round(((differences[itemName].meanAfter - differences[itemName].meanBefore) / differences[itemName].meanBefore) * 10000) / 100;
+      Math.round(
+        ((differences[itemName].meanAfter - differences[itemName].meanBefore) /
+          differences[itemName].meanBefore) *
+          10000
+      ) / 100;
   }
 
   // create a markdown table of the differences
   let markdown = `| Benchmark | Before | After | Change |\n`;
   markdown += `| :-- | --: | --: | --: |\n`;
+  let colors = "";
+
   for (const key in differences) {
     const diff = differences[key];
     let prependSign = "";
@@ -111,16 +131,26 @@ export async function compareBenchmarks(
     } else if (diff.meanDiff <= 0) {
       appendColor = "🟩";
     }
+
+    if (Math.abs(diff.meanDiff) <= diff.maxSD) {
+      appendColor = "⬜";
+    }
+
+    colors += appendColor;
+
     let changeText = !!diff.meanPercentDiff
-      ? `${prependSign}${fmtNum(diff.meanDiff, "ms")} (${prependSign}${fmtNum(diff.meanPercentDiff, "%")})${appendColor}`
+      ? `${prependSign}${fmtNum(diff.meanDiff, "ms")} (${prependSign}${fmtNum(
+          diff.meanPercentDiff,
+          "%"
+        )})${appendColor}`
       : "...";
 
     let beforeText = fmtNum(diff.meanBefore, "ms");
-    if(!isNaN(diff.meanBefore)) {
+    if (!isNaN(diff.meanBefore)) {
       beforeText += `±${fmtNum(diff.moeBefore)}`;
     }
     let afterText = fmtNum(diff.meanAfter, "ms");
-    if(!isNaN(diff.meanAfter)) {
+    if (!isNaN(diff.meanAfter)) {
       afterText += `±${fmtNum(diff.moeAfter)}`;
     }
 
@@ -147,16 +177,22 @@ export async function compareBenchmarks(
 ## Benchmarks
 
 <details>
-<summary>Results</summary>
+<summary>Comparison to Baseline ${colors}</summary>
 
-${createTable(targetResult[0])}
+${markdown}
+
+⬜ Within 1.5 standard deviations
+🟩 Faster, Above 1.5 standard deviations
+🟥 Slower, Above 1.5 standard deviations
+
+_Benchmarks may vary outside of normal expectations, especially when running in GitHub Actions CI._
 
 </details>
 
 <details>
-<summary>Comparison to ${previousSource}</summary>
+<summary>Results</summary>
 
-${markdown}
+${createTable(targetResult[0])}
 
 </details>
 

tools/hangar/src/package.setup.ts

@@ -19,6 +19,15 @@ const shellEnv = {
 };
 
 const getInstallArgs = async () => {
+  if(process.env.HANGAR_WINGLANG_PACKAGE) {
+    return [
+      "install",
+      "--no-package-lock",
+      "--install-links=false",
+      process.env.HANGAR_WINGLANG_PACKAGE!
+    ];
+  }
+
   if (process.env.CI) {
     const tarballsDir = path.resolve(`${__dirname}/../../../dist`);
     const tarballs = (await fs.readdir(tarballsDir))