[ci] Benchmark comment: show avg-latency deltas vs main (#2842)

Author: VaguelySerious

.changeset/wise-planes-tell.md

@@ -0,0 +1,2 @@
+---
+---

.github/scripts/render-benchmark-comment.mjs

@@ -13,6 +13,7 @@
  *   node render-benchmark-comment.mjs \
  *     --status running|completed|failed \
  *     [--results-dir <dir>]        # dir with bench-results-*.json files
+ *     [--baseline-dir <dir>]       # main-branch results to diff averages against
  *     [--previous-body <file>]     # previous comment body to carry history from
  *     [--commit <sha>] [--run-url <url>] \
  *     [--output <file>]            # defaults to stdout
@@ -48,6 +49,7 @@ export function parseArgs(argv) {
   const args = {
     status: 'completed',
     resultsDir: undefined,
+    baselineDir: undefined,
     previousBody: undefined,
     commit: undefined,
     runUrl: undefined,
@@ -66,6 +68,9 @@ export function parseArgs(argv) {
       case '--results-dir':
         args.resultsDir = next();
         break;
+      case '--baseline-dir':
+        args.baselineDir = next();
+        break;
       case '--previous-body':
         args.previousBody = next();
         break;
@@ -149,6 +154,50 @@ function formatMs(value) {
   return `${Math.abs(value) >= 100 ? Math.round(value) : value}`;
 }
 
+/**
+ * Annotates each metric row with the matching baseline average (from the most
+ * recent main-branch run), keyed by backend/app/metric/scenario. The
+ * annotation is stored on the entry so history re-renders keep showing the
+ * delta each run was originally compared against.
+ */
+export function annotateWithBaseline(results, baseline) {
+  if (!baseline || baseline.length === 0) return results;
+  const baselineAvgs = new Map();
+  for (const result of baseline) {
+    for (const row of result.metrics ?? []) {
+      baselineAvgs.set(
+        `${result.backend}/${result.app}/${row.metric}/${row.scenario}`,
+        row.avg
+      );
+    }
+  }
+  return results.map((result) => ({
+    ...result,
+    metrics: (result.metrics ?? []).map((row) => {
+      const baselineAvg = baselineAvgs.get(
+        `${result.backend}/${result.app}/${row.metric}/${row.scenario}`
+      );
+      return typeof baselineAvg === 'number' ? { ...row, baselineAvg } : row;
+    }),
+  }));
+}
+
+/** Formats the avg-vs-main delta, e.g. " (+4.2%)"; empty without a baseline. */
+function formatDelta(current, baselineAvg) {
+  if (
+    typeof current !== 'number' ||
+    typeof baselineAvg !== 'number' ||
+    baselineAvg <= 0 ||
+    !Number.isFinite(current / baselineAvg)
+  ) {
+    return '';
+  }
+  const pct = ((current - baselineAvg) / baselineAvg) * 100;
+  if (Math.abs(pct) < 0.5) return ' (±0%)';
+  const digits = Math.abs(pct) >= 10 ? 0 : 1;
+  return ` (${pct > 0 ? '+' : ''}${pct.toFixed(digits)}%)`;
+}
+
 /**
  * Formats a percentile cell, marking it 🟢/🔴 against its target when one is
  * defined for this row.
@@ -182,7 +231,7 @@ function renderResultTable(result) {
     const name = label ? `**${label.name}**` : row.metric;
     const targets = row.targets ?? {};
     lines.push(
-      `| ${name} | ${row.scenario} | ${formatMs(row.avg)} | ${formatCell(row.p75, targets.p75)} | ${formatCell(row.p90, targets.p90)} | ${formatCell(row.p99, targets.p99)} | ${row.samples} |`
+      `| ${name} | ${row.scenario} | ${formatMs(row.avg)}${formatDelta(row.avg, row.baselineAvg)} | ${formatCell(row.p75, targets.p75)} | ${formatCell(row.p90, targets.p90)} | ${formatCell(row.p99, targets.p99)} | ${row.samples} |`
     );
   }
   return lines.join('\n');
@@ -249,8 +298,17 @@ function renderFooter(entries) {
   ).join(' · ');
   const scenarioLegend = buildScenarioLegend(results);
   const targetsLegend = buildTargetsLegend(results);
+  const hasBaseline = results.some((result) =>
+    (result.metrics ?? []).some((row) => typeof row.baselineAvg === 'number')
+  );
 
   return [
+    ...(hasBaseline
+      ? [
+          '<sub>Avg deltas compare against the most recent benchmark run on `main` at the time of this run.</sub>',
+          '',
+        ]
+      : []),
     `<sub>Metrics — ${definitions}</sub>`,
     ...(scenarioLegend ? ['', `<sub>Scenarios — ${scenarioLegend}</sub>`] : []),
     ...(targetsLegend
@@ -312,6 +370,7 @@ function renderHistorySection(shownPrevious) {
 export function renderComment({
   status,
   results,
+  baseline = [],
   history,
   commit,
   runUrl,
@@ -324,7 +383,7 @@ export function renderComment({
         commit,
         runUrl,
         generatedAt: now.toISOString(),
-        results,
+        results: annotateWithBaseline(results, baseline),
       },
       ...entries,
     ].slice(0, MAX_HISTORY_ENTRIES);
@@ -367,6 +426,10 @@ function main() {
     : '';
   const history = extractHistory(previousBody);
   const results = loadResults(args.resultsDir);
+  const baseline = loadResults(args.baselineDir);
+  if (args.baselineDir && baseline.length === 0) {
+    console.error(`No baseline results found in ${args.baselineDir}`);
+  }
 
   if (args.status === 'completed' && results.length === 0) {
     console.error('No benchmark results found for status=completed');
@@ -376,6 +439,7 @@ function main() {
   const body = renderComment({
     status: args.status,
     results,
+    baseline,
     history,
     commit: args.commit,
     runUrl: args.runUrl,

.github/scripts/render-benchmark-comment.test.js

@@ -136,6 +136,59 @@ test('renders a completed run with a table and embedded history', async () => {
   assert.strictEqual(history[0].results[0].metrics.length, 4);
 });
 
+test('renders avg deltas against a baseline and embeds them in history', async () => {
+  const { renderComment, extractHistory } = await loadModule();
+  const baseline = sampleResult({
+    metrics: sampleResult()
+      .metrics.filter((row) => row.metric !== 'wo') // no baseline for WO
+      .map((row) => ({
+        ...row,
+        // ttfs 412.3 vs 400 → +3.1%; sl 55.1 vs 50 → +10%; stso 91 vs 91 → ±0%
+        avg: { ttfs: 400, sl: 50, stso: 91 }[row.metric],
+      })),
+  });
+  const body = renderComment({
+    status: 'completed',
+    results: [sampleResult()],
+    baseline: [baseline],
+    history: [],
+    commit: 'abcdef1234567890',
+  });
+
+  // 412.3 renders rounded (>= 100), 55.1 keeps its decimal
+  assert.match(body, /\| 412 \(\+3\.1%\) \|/);
+  assert.match(body, /\| 55\.1 \(\+10%\) \|/);
+  assert.match(body, /\| 91 \(±0%\) \|/);
+  // WO has no baseline row → no delta
+  assert.match(body, /\| 1200 \|/);
+  assert.match(
+    body,
+    /Avg deltas compare against the most recent benchmark run on `main`/
+  );
+  // The annotation is embedded so history re-renders keep the delta
+  const history = extractHistory(body);
+  assert.strictEqual(history[0].results[0].metrics[0].baselineAvg, 400);
+  const rerendered = renderComment({
+    status: 'running',
+    results: [],
+    history,
+    commit: 'ffffff1234567890',
+  });
+  assert.match(rerendered, /\| 412 \(\+3\.1%\) \|/);
+});
+
+test('renders no deltas without a baseline', async () => {
+  const { renderComment } = await loadModule();
+  const body = renderComment({
+    status: 'completed',
+    results: [sampleResult()],
+    history: [],
+    commit: 'abcdef1234567890',
+  });
+  assert.doesNotMatch(body, /%\)/);
+  assert.doesNotMatch(body, /Avg deltas/);
+});
+
 test('collapses previous results on re-runs', async () => {
   const { renderComment, extractHistory } = await loadModule();
   const first = renderComment({
@@ -259,13 +312,35 @@ test('CLI renders results from a directory and previous body file', async () =>
   const first = fs.readFileSync(firstOut, 'utf8');
   assert.match(first, /398 🔴/);
 
+  // Baseline files arrive nested in per-artifact subdirectories (that's how
+  // the download action extracts them); loadResults must find them anyway.
+  const baselineDir = path.join(dir, 'baseline');
+  fs.mkdirSync(
+    path.join(baselineDir, 'bench-results-nextjs-turbopack-vercel'),
+    {
+      recursive: true,
+    }
+  );
+  const baseline = sampleResult();
+  baseline.metrics = baseline.metrics.map((row) => ({ ...row, avg: 400 }));
+  fs.writeFileSync(
+    path.join(
+      baselineDir,
+      'bench-results-nextjs-turbopack-vercel',
+      'bench-results-nextjs-turbopack-vercel.json'
+    ),
+    JSON.stringify(baseline)
+  );
+
   const secondOut = path.join(dir, 'comment2.md');
   execFileSync(process.execPath, [
     SCRIPT,
     '--status',
     'completed',
     '--results-dir',
     resultsDir,
+    '--baseline-dir',
+    baselineDir,
     '--previous-body',
     firstOut,
     '--commit',
@@ -276,6 +351,8 @@ test('CLI renders results from a directory and previous body file', async () =>
   const second = fs.readFileSync(secondOut, 'utf8');
   assert.match(second, /Previous results \(1\)/);
   assert.match(second, /#### 1111111/);
+  // ttfs avg 412.3 (rendered rounded) vs baseline 400 → +3.1%
+  assert.match(second, /\| 412 \(\+3\.1%\) \|/);
 });
 
 test('CLI fails when completed with no results', async () => {

.github/workflows/benchmarks.yml

@@ -5,10 +5,16 @@ name: Performance Benchmarks
 # workbench app and posts the results as a sticky PR comment. Re-runs update
 # the same comment; previous results stay available in a collapsed history
 # section (state is embedded in the comment body itself).
+#
+# Runs on pushes to main (against the production deployment) purely to produce
+# baseline artifacts; PR runs download the most recent main baseline and show
+# avg-latency deltas against it in the comment.
 
 on:
   pull_request:
     branches: [main]
+  push:
+    branches: [main]
   workflow_dispatch: # Allow manual triggers
 
 concurrency:
@@ -193,6 +199,36 @@ jobs:
           path: benchmark-results
           merge-multiple: true
 
+      # Baseline for the avg-delta column: results from the most recent
+      # successful run of this workflow on main. Uses gh (first-party) rather
+      # than a third-party cross-run artifact action. Filtering to push /
+      # workflow_dispatch events matters for more than tidiness: it excludes
+      # artifacts uploaded by fork PR runs, whose head branch can also be
+      # named "main" (artifact poisoning).
+      - name: Download baseline results from main
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          run_id=$(gh run list \
+            --repo "$GITHUB_REPOSITORY" \
+            --workflow benchmarks.yml \
+            --branch main \
+            --status success \
+            --limit 20 \
+            --json databaseId,event \
+            --jq '[.[] | select(.event == "push" or .event == "workflow_dispatch")][0].databaseId // empty')
+          if [ -z "$run_id" ]; then
+            echo "No successful benchmark run found on main; skipping baseline"
+            exit 0
+          fi
+          echo "Using baseline artifacts from run $run_id"
+          gh run download "$run_id" \
+            --repo "$GITHUB_REPOSITORY" \
+            --pattern 'bench-results-*' \
+            --dir baseline-results \
+            || echo "Run $run_id has no benchmark artifacts; skipping baseline"
+
       - name: Find existing benchmark comment
         uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: find-comment
@@ -222,6 +258,7 @@ jobs:
           node .github/scripts/render-benchmark-comment.mjs \
             --status "$BENCH_STATUS" \
             --results-dir benchmark-results \
+            --baseline-dir baseline-results \
             --previous-body "$RUNNER_TEMP/previous-comment.md" \
             --commit "$HEAD_SHA" \
             --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \