bench: per-chunk stream latency (CRTT/CDV) and replay-driven stream scenarios

.changeset/bench-chunk-rtt-scenarios.md

@@ -0,0 +1,4 @@
+---
+---
+
+Add per-chunk stream latency to the CI benchmark: CRTT/CDV metrics with a paced control, a size sweep, and replay scenarios driven by real captured cadences at the eve and AI-gateway boundaries, reported in a dedicated Streams table plus a pooled first-chunk RTT row; the SL/SO report rows are retired (CRTT subsumes both).

.github/scripts/render-benchmark-comment.test.js

@@ -23,6 +23,17 @@ function sampleResult(overrides = {}) {
       sequentialIterations: 1,
       sequentialStepCount: 1020,
       warmupIterations: 2,
+      replayCadences: [
+        {
+          id: 'eve-test-cadence',
+          model: 'test-model',
+          events: 823,
+          spanMs: 6196,
+          totalBytes: 2000000,
+          semanticSha256:
+            '609bc99fb5eb810086dcaecc9128f5fecd7c75d8bc3f2b39a6622f89d5a5a47a',
+        },
+      ],
     },
     scenarios: [
       { name: 'stream', description: 'one streaming step in turbo mode' },
@@ -120,6 +131,11 @@ test('renders a completed run with a table and embedded history', async () => {
     body,
     /<sub>Scenarios — \*\*stream\*\*: one streaming step in turbo mode/
   );
+  // Replay-cadence identity line: full semantic hash on its own legend line
+  assert.match(
+    body,
+    /<sub>Replay cadences \(semantic sha256\) — \*\*eve-test-cadence\*\* `609bc99fb5eb810086dcaecc9128f5fecd7c75d8bc3f2b39a6622f89d5a5a47a`<\/sub>/
+  );
   // Target marks: TTFS p75 398 > 200 → 🔴; SL row is within target on every
   // percentile, so it stays unmarked (no 🟢 anywhere); WO has no targets.
   assert.match(body, /398 🔴/);
@@ -580,6 +596,210 @@ function sequentialResult({ inline, queueHop }) {
   });
 }
 
+// Fixed log-bin edges matching RTT_HIST_EDGES_MS in the bench helper module
+// (workbench/example/workflows/97_bench_rtt.ts).
+const CRTT_EDGES = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000];
+
+/** Histogram over CRTT_EDGES with counts placed by (value, count) pairs. */
+function crttHist(entries) {
+  const counts = new Array(CRTT_EDGES.length + 1).fill(0);
+  for (const [value, count] of entries) {
+    let bin = 0;
+    while (bin < CRTT_EDGES.length && value >= CRTT_EDGES[bin]) bin++;
+    counts[bin] += count;
+  }
+  return counts;
+}
+
+function crttResult({ avg = 120, hist }) {
+  const streamRow = (scenario, group, extra = {}) => ({
+    metric: 'stream',
+    scenario,
+    unit: 'ms',
+    best: 59,
+    avg,
+    p50: 128,
+    p75: 188,
+    p90: 438,
+    p99: 1229,
+    samples: hist.reduce((a, b) => a + b, 0),
+    raw: [],
+    hist: { edgesMs: CRTT_EDGES, counts: hist },
+    group,
+    bucket: 'all',
+    stream: {
+      iterations: 10,
+      wrCps: 100,
+      wrKiBps: 6.1,
+      rdCps: 99.4,
+      rdKiBps: 6,
+      firstMs: 96,
+      cdvMaxMs: 141,
+      runs: [
+        { wrCps: 100, rdCps: 99.4, firstMs: 96, cdvMaxMs: 141, slipMaxMs: 4 },
+      ],
+    },
+    ...extra,
+  });
+  return sampleResult({
+    scenarios: [
+      { name: 'chunk RTT (llm)', description: 'self-timestamping chunks' },
+    ],
+    metrics: [
+      streamRow('chunk RTT (llm)', 'llm', {
+        progressAvgMs: [110, 112, 115, 113, 118, 120, 119, 125, 130, 135],
+        cdvAvgMs: [2, 2, 3, 5, 9, 15, 24, 40, 66, 108],
+      }),
+      // Artifact-only detail rows: per-index CRTT split and slip tail.
+      {
+        metric: 'crtt',
+        scenario: 'chunk RTT llm (seq 0)',
+        unit: 'ms',
+        best: 97,
+        avg: 130,
+        p50: 112,
+        p75: 126,
+        p90: 129,
+        p99: 157,
+        samples: 10,
+        raw: [],
+        group: 'llm',
+        bucket: 'seq 0',
+        detail: true,
+      },
+      {
+        metric: 'slip',
+        scenario: 'write slip (llm)',
+        unit: 'ms',
+        best: 2,
+        avg: 3,
+        p50: 3,
+        p75: 4,
+        p90: 5,
+        p99: 6,
+        samples: 10,
+        raw: [],
+        group: 'llm',
+        detail: true,
+      },
+      streamRow('replay eve-test (2x)', 'replay', {
+        stream: {
+          iterations: 5,
+          wrCps: 297,
+          wrKiBps: 742,
+          rdCps: 288,
+          rdKiBps: 719,
+          firstMs: 118,
+          cdvMaxMs: 210,
+          runs: [
+            {
+              wrCps: 297,
+              rdCps: 288,
+              firstMs: 118,
+              cdvMaxMs: 210,
+              slipMaxMs: 9,
+            },
+          ],
+        },
+      }),
+    ],
+  });
+}
+
+test('renders stream scenarios in their own table with rate columns', async () => {
+  const { renderComment, extractHistory } = await loadModule();
+  const hist = crttHist([
+    [59, 1400],
+    [128, 1500],
+    [438, 100],
+  ]);
+  const baseline = crttResult({ avg: 150, hist });
+  // Baseline medians differ so deltas render: rd rate was lower on main.
+  baseline.metrics[0].stream.rdCps = 90;
+  const body = renderComment({
+    status: 'completed',
+    results: [crttResult({ avg: 120, hist })],
+    baseline: [baseline],
+    history: [],
+    commit: 'abcdef1234567890',
+  });
+
+  // Stream rows are OUT of the metric table and IN the Streams table.
+  assert.doesNotMatch(body, /\| \*\*stream\*\* \|/);
+  assert.match(
+    body,
+    /\| Scenario \| wr c\/s \| rd c\/s \| wr KiB\/s \| rd KiB\/s \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/
+  );
+  // Rate cells with plain vs-main deltas, latency cells from percentile
+  // baselines, and NO red/green marks anywhere in the stream table.
+  assert.match(
+    body,
+    /\| chunk RTT \(llm\) \| 100 \(\u00b10%\) \| 99\.4 \(\+10%\) \| 6\.1 \(\u00b10%\) \|/
+  );
+  assert.match(
+    body,
+    /\| replay eve-test \(2x\) \| 297 \(\u00b10%\) \| 288 \(\u00b10%\) \| 742 \(\u00b10%\) \| 719 \(\u00b10%\) \|/
+  );
+  assert.match(body, /\| 141 \(\u00b10%\) \| 10 \|/);
+  const streamsSection = body.slice(
+    body.indexOf('**Streams**'),
+    body.indexOf('</details>')
+  );
+  assert.doesNotMatch(
+    streamsSection,
+    /\ud83d\udd34|\ud83d\udfe2|\ud83d\udd3b|\ud83d\udc9a/
+  );
+  // Detail rows render nowhere.
+  assert.doesNotMatch(body, /seq 0 \|/);
+  assert.doesNotMatch(body, /write slip/);
+  // Drill-down still renders from the stream rows.
+  assert.match(body, /\ud83d\udcc8 CRTT drill-down/);
+  assert.match(
+    body,
+    /llm +\u00b7+[\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588]*\u2588/
+  );
+  assert.match(body, /Delivery jitter over stream progress/);
+  // CRTT + CDV definitions stay in the legend (stream table columns), and
+  // the internal 'stream' id never leaks into it.
+  assert.match(body, /\*\*CRTT\*\*: chunk round-trip time/);
+  assert.match(body, /\*\*CDV\*\*: chunk delay variation/);
+  assert.match(body, /\*\*Streams\*\*: writer\/reader sustained rates/);
+  // History block: per-run arrays and sparkline payloads stripped, medians
+  // and baseline annotations kept.
+  const history = extractHistory(body);
+  const kept = history[0].results[0].metrics[0];
+  assert.strictEqual(kept.hist, undefined);
+  assert.strictEqual(kept.progressAvgMs, undefined);
+  assert.strictEqual(kept.stream.runs, undefined);
+  assert.strictEqual(kept.stream.wrCps, 100);
+  assert.strictEqual(kept.baselineStream.rdCps, 90);
+  // Re-render from history keeps the Streams table, drops the drill-down.
+  const rerendered = renderComment({
+    status: 'running',
+    results: [],
+    history,
+    commit: 'ffffff1234567890',
+  });
+  assert.match(rerendered, /\| chunk RTT \(llm\) \| 100/);
+  assert.doesNotMatch(rerendered, /CRTT drill-down/);
+});
+
+test('renders the stream table without deltas when main has no baseline', async () => {
+  const { renderComment } = await loadModule();
+  const body = renderComment({
+    status: 'completed',
+    results: [crttResult({ hist: crttHist([[128, 3000]]) })],
+    history: [],
+    commit: 'abcdef1234567890',
+  });
+  assert.match(
+    body,
+    /\| chunk RTT \(llm\) \| 100 \| 99\.4 \| 6\.1 \| 6 \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/
+  );
+  assert.doesNotMatch(body, /%\)/);
+  assert.match(body, /No `main` baseline yet/);
+});
+
 test('renders inline and queue-hop STSO histogram diffs against main', async () => {
   const { renderComment } = await loadModule();
   const body = renderComment({

.github/workflows/benchmarks.yml

@@ -1,8 +1,8 @@
 name: Performance Benchmarks
 
-# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, SL —
-# see packages/core/e2e/benchmark.test.ts for definitions) against a deployed
-# workbench app and posts the results as a sticky PR comment. Re-runs update
+# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, CRTT,
+# CDV — see packages/core/e2e/benchmark.test.ts for definitions) against a
+# deployed workbench app and posts the results as a sticky PR comment. Re-runs update
 # the same comment; previous results stay available in a collapsed history
 # section (state is embedded in the comment body itself).
 #
@@ -96,10 +96,10 @@ jobs:
   # backend (e.g. postgres or local), add a matrix entry with
   # `world: postgres` / `world: local` and gate the Vercel-specific steps —
   # the runner (packages/core/e2e/benchmark.test.ts) already selects its
-  # backend from the same env vars as the e2e tests. SL is measured inside the
-  # workflow (benchSlWorkflow's parallel reader/writer steps read/write on the
-  # deployment), so it no longer needs `run.getReadable()` to work from the
-  # test process; the runner only polls returnValue for the collected timings.
+  # backend from the same env vars as the e2e tests. CRTT is measured inside
+  # the workflow (benchCrttWorkflow's parallel reader/writer steps read/write
+  # on the deployment), so it does not need `run.getReadable()` to work from
+  # the test process; the runner only polls returnValue for the aggregates.
   benchmark:
     name: Benchmark (${{ matrix.target.world }}, ${{ matrix.target.app }})
     runs-on: ubuntu-latest