From c1cf1e30a103d5b61d57fa193b2439558bc4fb20 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Thu, 13 Aug 2026 16:25:13 -0500 Subject: [PATCH 1/3] bench: per-chunk stream latency (CRTT/CDV) and replay-driven stream scenarios Adds per-chunk stream latency to the CI benchmark report: CRTT (chunk round-trip time, same-clock write->read per chunk, aggregated inside the reader step into fixed log-bin histograms, index buckets, and mean-RTT profiles), CDV (chunk delay variation - skew-free delivery jitter, each run's max positive value), a pooled first-chunk RTT row (the stream-open path; exact percentiles), and replay scenarios driven by real captured cadences at the eve (envelope-protocol outlier) and AI-gateway (typical customer) boundaries - workload measured, not invented; the 2x speed multiplier on the eve stress row is the only chosen parameter. Stream scenarios render in their own table (rates, first-chunk RTT, CRTT percentiles, CDV max; medians across iterations, no targets yet). SL/SO report rows are retired: CRTT's seq-0 slice reproduces SL and its aggregate reproduces SO at ~100x the samples; write slip stays as artifact-only producer-stall data. Captures carry semantic sha256 hashes over canonical event tuples for cross-system identity with durabench. Signed-off-by: Alex Langenfeld --- .github/scripts/render-benchmark-comment.mjs | 416 ++++++++++- .../scripts/render-benchmark-comment.test.js | 220 ++++++ .github/workflows/benchmarks.yml | 14 +- packages/core/e2e/benchmark.test.ts | 700 ++++++++++++++---- .../core/src/bench-chunk-rtt-stats.test.ts | 377 ++++++++++ workbench/example/workflows/97_bench.ts | 417 +++++++++++ .../example/workflows/97_bench_cadence.ts | 75 ++ workbench/example/workflows/97_bench_rtt.ts | 390 ++++++++++ .../workflows/97_bench_cadence.ts | 1 + .../workflows/97_bench_rtt.ts | 1 + .../nitro-v3/workflows/97_bench_cadence.ts | 1 + workbench/nitro-v3/workflows/97_bench_rtt.ts | 1 + 12 files changed, 2432 insertions(+), 181 deletions(-) create mode 100644 packages/core/src/bench-chunk-rtt-stats.test.ts create mode 100644 workbench/example/workflows/97_bench_cadence.ts create mode 100644 workbench/example/workflows/97_bench_rtt.ts create mode 120000 workbench/nextjs-turbopack/workflows/97_bench_cadence.ts create mode 120000 workbench/nextjs-turbopack/workflows/97_bench_rtt.ts create mode 120000 workbench/nitro-v3/workflows/97_bench_cadence.ts create mode 120000 workbench/nitro-v3/workflows/97_bench_rtt.ts diff --git a/.github/scripts/render-benchmark-comment.mjs b/.github/scripts/render-benchmark-comment.mjs index 51af3ec75d..eb25f7f588 100644 --- a/.github/scripts/render-benchmark-comment.mjs +++ b/.github/scripts/render-benchmark-comment.mjs @@ -31,7 +31,7 @@ const METRIC_LABELS = { ttfs: { name: 'TTFS', description: - 'time to first step body (in-deployment start() → first step body, deployment clocks)', + 'time to first step body (in-deployment start() → first step body)', }, 'fanout-ttfs': { name: 'Fan-out TTFS', @@ -62,6 +62,26 @@ const METRIC_LABELS = { description: 'stream overhead (end-to-end write+consume time beyond the modelled generation window)', }, + // Name reservations: CTT = future production one-way write→read metric + // (cross-clock); TTFC = future consumer-journey start → first-chunk-readable + // metric. The 'first chunk (pooled)' row is neither (readAt₀ - writtenAt₀, + // a round trip), so it stays under CRTT. + crtt: { + name: 'CRTT', + description: + 'chunk round-trip time (per-chunk write → read latency, one clock domain: deployment → stream backend → same deployment)', + }, + cdv: { + name: 'CDV', + description: + "chunk delay variation / delivery jitter (inter-arrival gap minus inter-write gap per seq-adjacent pair; skew-free; the row is each run's MAX positive value, so one stall moves it)", + }, + slip: { + // Title-case on purpose (a word, not an initialism). Artifact-only. + name: 'Slip', + description: + "write slip (how late each chunk was written vs the writer's open-loop schedule; the row is each run's MAX — the producer-stall guard that RTT and CDV both hide)", + }, }; const METRIC_ORDER = [ 'ttfs', @@ -71,6 +91,9 @@ const METRIC_ORDER = [ 'wo', 'sl', 'so', + 'crtt', + 'cdv', + 'slip', ]; export function parseArgs(argv) { @@ -138,17 +161,19 @@ export function extractHistory(body) { } /** - * Drops the per-metric raw sample arrays before embedding an entry in the - * comment's data block. The sequential-steps scenario records ~1000 STSO - * samples per run (plus the baseline's), which would blow past GitHub's - * comment size limit within a couple of history entries; the percentiles and - * baseline annotations — everything the history tables render — are kept. + * Drops the per-metric raw sample arrays (and the CRTT fixed-bin histograms) + * before embedding an entry in the comment's data block. The sequential-steps + * scenario records ~1000 STSO samples per run (plus the baseline's), which + * would blow past GitHub's comment size limit within a couple of history + * entries; the percentiles and baseline annotations — everything the history + * tables render — are kept. * - * This does not affect the histogram diff against `main`: that reads its - * baseline from the artifacts the workflow downloads into --baseline-dir, - * which keep their raw samples. What it costs is the collapsed "Previous - * results" entries, re-rendered from this block on a later commit of the same - * PR — they show their tables but not their histograms. + * This does not affect the distribution diffs against `main`: those read + * their baselines from the artifacts the workflow downloads into + * --baseline-dir, which keep raw samples and histograms. What it costs is the + * collapsed "Previous results" entries, re-rendered from this block on a + * later commit of the same PR — they show their tables but not their + * histograms. */ function stripRawSamples(entries) { return entries.map((entry) => ({ @@ -156,7 +181,23 @@ function stripRawSamples(entries) { results: (entry.results ?? []).map((result) => ({ ...result, metrics: (result.metrics ?? []).map( - ({ raw, baselineRaw, ...row }) => row + ({ + raw, + baselineRaw, + hist, + progressAvgMs, + sizeAvgMs, + cdvAvgMs, + ...row + }) => { + // Stream rows: keep the median columns for history tables, drop + // the per-run arrays. + if (row.stream?.runs) { + const { runs, ...medians } = row.stream; + return { ...row, stream: medians }; + } + return row; + } ), })), })); @@ -228,6 +269,10 @@ const BASELINE_FIELDS = [ { annotation: 'baselineP75', from: (base) => base.p75 }, { annotation: 'baselineP90', from: (base) => base.p90 }, { annotation: 'baselineP99', from: (base) => base.p99 }, + // Not rendered in the main table (no Avg/P50 columns there), but the CRTT + // drill-down matrix shows vs-main deltas on both (avg deltas are exact). + { annotation: 'baselineAvg', from: (base) => base.avg }, + { annotation: 'baselineP50', from: (base) => base.p50 }, ]; export function annotateWithBaseline(results, baseline) { @@ -253,6 +298,12 @@ export function annotateWithBaseline(results, baseline) { // histogram diff below the table — kept separate from BASELINE_FIELDS // since it's an array, not a numeric percentile. if (Array.isArray(base.raw)) annotated.baselineRaw = base.raw; + // Stream rows diff their rate/CDV columns against the baseline's stream + // object (per-run arrays dropped — only the medians are compared). + if (row.stream && base.stream) { + const { runs, ...medians } = base.stream; + annotated.baselineStream = medians; + } return annotated; }; return results.map((result) => ({ @@ -389,8 +440,13 @@ function renderOverlayBar(base, cur, maxCount) { * and their delta on the same line (a fenced code block keeps everything * aligned in a monospace font). This is the whole histogram diff — the shape * of the two distributions and the per-bucket numbers behind it, without a - * second table restating them. */ -function renderStsoBarChart(buckets, { selfDiff } = {}) { + * second table restating them. Shared by the STSO section (buckets = step + * counts) and the CRTT section (buckets = chunk counts); `selfLabel` names + * the series when there is no baseline to overlay. */ +function renderHistogramBarChart( + buckets, + { selfDiff, selfLabel = 'steps' } = {} +) { const maxCount = Math.max(1, ...buckets.map((b) => Math.max(b.base, b.cur))); const labelWidth = Math.max(...buckets.map((b) => b.label.length)); const countWidth = Math.max( @@ -408,7 +464,7 @@ function renderStsoBarChart(buckets, { selfDiff } = {}) { : renderOverlayBar(base, cur, maxCount) ).padEnd(BAR_CHART_WIDTH); const counts = selfDiff - ? `steps ${String(cur).padStart(countWidth)}` + ? `${selfLabel} ${String(cur).padStart(countWidth)}` : `main ${String(base).padStart(countWidth)} this ${String(cur).padStart(countWidth)} ${formatDeltaValue(cur - base).padStart(countWidth + 1)}`; lines.push(`${label.padStart(labelWidth)} ms ${bar} ${counts}`); } @@ -464,7 +520,7 @@ function renderStsoRowDiff(row) { ); } if (buckets.length > 0) { - lines.push(renderStsoBarChart(buckets, { selfDiff })); + lines.push(renderHistogramBarChart(buckets, { selfDiff })); } return lines.join('\n'); } @@ -498,6 +554,166 @@ function renderStsoDiffSection(result) { ].join('\n'); } +// ============================================================================ +// CRTT drill-down (per-bucket sparkline matrix, vs main) +// ============================================================================ + +const SPARK_LEVELS = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; + +/** One-character-per-bin sparkline over fixed histogram counts, normalized to + * the row's own max so every bucket's *shape* is readable regardless of its + * sample count. Empty bins render as `·` so the fixed log axis stays visible + * and the occupied bins' *position* on it (fast vs slow) is comparable across + * lines. */ +function sparkline(counts) { + const max = Math.max(1, ...counts); + return counts + .map((c) => + c === 0 + ? '·' + : SPARK_LEVELS[ + Math.min( + SPARK_LEVELS.length - 1, + Math.floor((c / max) * SPARK_LEVELS.length) + ) + ] + ) + .join(''); +} + +/** + * Renders the CRTT drill-down: ONE line per variant — a sparkline of the + * fixed log-bin RTT histogram plus avg/p50/p90/p99 (plain vs-main + * percentages when a baseline exists) — followed by the mean-RTT profile + * lines. Per-index detail rows are deliberately NOT rendered: three runs + * showed them flat and their run-to-run flips are bucket-hopping iteration + * noise that invites misreads. They stay in the results JSON (with baseline + * annotations), so when a headline delta fires the artifact still localizes + * it; the progress line guards position-dependence here with finer + * resolution than the buckets did. + * + * The avg deltas are exact (count-weighted merges on both sides); p50-p99 + * are cross-iteration percentile-of-percentiles, like the main table. + * Collapsed by default, like the STSO section: a drill-down, not the + * headline. + */ +function renderCrttMatrixSection(result) { + const rows = (result.metrics ?? []).filter( + (row) => row.stream && !row.detail && Array.isArray(row.hist?.counts) + ); + if (rows.length === 0) return ''; + const anyBaseline = rows.some((row) => typeof row.baselineAvg === 'number'); + + const round1 = (v) => Math.round(v * 10) / 10; + const pct = (cur, base) => { + if (typeof cur !== 'number' || typeof base !== 'number' || base <= 0) { + return ''; + } + const p = ((cur - base) / base) * 100; + if (Math.abs(p) < 0.5) return ' (±0%)'; + return ` (${p > 0 ? '+' : ''}${Math.round(p)}%)`; + }; + const cells = (row) => [ + row.group ?? row.scenario, + sparkline(row.hist.counts), + `${round1(row.avg)}${pct(row.avg, row.baselineAvg)}`, + `${formatMs(row.p50)}${pct(row.p50, row.baselineP50)}`, + `${formatMs(row.p90)}${pct(row.p90, row.baselineP90)}`, + `${formatMs(row.p99)}${pct(row.p99, row.baselineP99)}`, + String(row.samples), + ]; + const header = ['variant', 'RTT 1ms→5s+', 'avg', 'p50', 'p90', 'p99', 'n']; + const table = [header, ...rows.map(cells)]; + const widths = header.map((_, col) => + Math.max(...table.map((line) => line[col].length)) + ); + const renderLine = (line) => + line + .map((cell, col) => + // Left-align the label and sparkline columns, right-align numbers. + col <= 1 ? cell.padEnd(widths[col]) : cell.padStart(widths[col]) + ) + .join(' ') + .trimEnd(); + + const lines = ['```', renderLine(header)]; + for (const row of rows) { + lines.push(renderLine(cells(row))); + } + lines.push('```'); + + // Mean-RTT profile lines, one per variant that recorded the profile: + // - progress (per tenth of the stream): the drift readout — a rising + // staircase means chunks get slower as the stream grows, which fixed + // index buckets can't localize. + // - size (per log size bin, sweep only): the size→latency curve — flat + // means chunk size doesn't matter, a knee localizes where it starts to. + // Bars are scaled min→max per line so the *shape* stays readable even for + // small effects; the ms range alongside is what says whether the shape + // matters. Null entries (empty bins) render as `·`. + const renderProfileBlock = (title, entries) => { + if (entries.length === 0) return; + const labelWidth = Math.max(...entries.map((e) => e.label.length)); + lines.push('', title, '', '```'); + for (const { label, avgs } of entries) { + const present = avgs.filter((v) => typeof v === 'number'); + if (present.length === 0) continue; + const min = Math.min(...present); + const max = Math.max(...present); + const span = max - min; + const bars = avgs + .map((v) => + typeof v !== 'number' + ? '·' + : span <= 0 + ? SPARK_LEVELS[0] + : SPARK_LEVELS[ + Math.round(((v - min) / span) * (SPARK_LEVELS.length - 1)) + ] + ) + .join(''); + const range = `${Math.round(min)}–${Math.round(max)}ms`; + lines.push(`${label.padEnd(labelWidth)} ${bars} ${range}`); + } + lines.push('```'); + }; + const profileEntries = (field) => + rows + .filter((row) => Array.isArray(row[field]) && row[field].length > 0) + .map((row) => ({ label: row.group ?? row.scenario, avgs: row[field] })); + renderProfileBlock( + 'RTT over stream progress (avg per tenth of stream, bars scaled min→max):', + profileEntries('progressAvgMs') + ); + renderProfileBlock( + 'RTT by chunk size (avg per log size bin, ~160B → ~12KB serialized, bars scaled min→max):', + profileEntries('sizeAvgMs') + ); + // Where in the stream delivery clumping/stalls concentrate — the CDV + // row's per-run max says the worst stall's size; this says where. Flat is + // steady-cadence clumping; a hot spot localizes a stall. + renderProfileBlock( + 'Delivery jitter over stream progress (avg positive CDV per tenth of stream, bars scaled min→max):', + profileEntries('cdvAvgMs') + ); + + return [ + '', + '
', + `📈 CRTT drill-down${anyBaseline ? ' vs main' : ''} (RTT distributions & profiles)`, + '', + ...(anyBaseline + ? [] + : [ + 'No `main` baseline yet — percentages appear once a run on `main` has recorded CRTT.', + '', + ]), + lines.join('\n'), + '', + '
', + ].join('\n'); +} + // Deltas beyond ±this vs main get a directional marker: 🔻 for a regression, // 💚 for an improvement. Smaller moves show the percentage alone. const DELTA_MARK_THRESHOLD_PCT = 15; @@ -542,19 +758,83 @@ function shortCommit(commit) { return commit ? commit.slice(0, 7) : 'unknown'; } +// CDV (and, in older history entries, Slip) is the companion of CRTT, +// measured by the same scenario runs (same workload, same iterations). The +// table pairs each variant's rows — chunk RTT (llm) directly above delivery +// jitter (llm) — instead of grouping metric by metric. +const PAIRED_METRICS = { cdv: 'crtt', slip: 'crtt' }; + function metricSortKey(row) { - const idx = METRIC_ORDER.indexOf(row.metric); + const idx = METRIC_ORDER.indexOf(PAIRED_METRICS[row.metric] ?? row.metric); return idx === -1 ? METRIC_ORDER.length : idx; } +/** Orders rows within a paired-metric family: by variant (`group`, falling + * back to scenario for rows recorded before `group` existed), then anchor + * metric before companion. Non-family rows keep insertion order (0 preserves + * the stable sort). */ +function pairedSortKey(a, b) { + const inFamily = (metric) => + metric in PAIRED_METRICS || Object.values(PAIRED_METRICS).includes(metric); + if (!inFamily(a.metric) || !inFamily(b.metric)) return 0; + return ( + (a.group ?? a.scenario).localeCompare(b.group ?? b.scenario) || + METRIC_ORDER.indexOf(a.metric) - METRIC_ORDER.indexOf(b.metric) + ); +} + +/** + * The stream-scenario table: one row per stream scenario, with the columns + * streams actually want — writer-side achieved and reader-side delivered + * sustained rates (steady window: first/last 10% of chunks trimmed), CRTT + * percentiles, and the median worst delivery stall (CDV max positive). + * Deltas vs main are plain percentages; deliberately NO 🔴/🟢 marks — targets + * attach in a later PR once a baseline exists. Rates read higher-is-better, + * latencies lower-is-better, so directional marks would need per-column + * polarity anyway; numbers + deltas keep it honest until then. + */ +function renderStreamTable(result) { + const rows = (result.metrics ?? []).filter( + (row) => row.stream && !row.detail + ); + if (rows.length === 0) return ''; + const pct = (cur, base) => { + if (typeof cur !== 'number' || typeof base !== 'number' || base <= 0) { + return ''; + } + const p = ((cur - base) / base) * 100; + if (Math.abs(p) < 0.5) return ' (±0%)'; + return ` (${p > 0 ? '+' : ''}${Math.round(p)}%)`; + }; + const cell = (value, base) => + typeof value === 'number' ? `${formatMs(value)}${pct(value, base)}` : '—'; + const lines = [ + '**Streams**', + '', + '| Scenario | wr c/s | rd c/s | wr KiB/s | rd KiB/s | CRTT 1st | p75 | p90 | p99 | CDV max | iters |', + '|----------|-------:|-------:|---------:|---------:|---------:|----:|----:|----:|--------:|------:|', + ]; + for (const row of rows) { + const s = row.stream; + const b = row.baselineStream ?? {}; + lines.push( + `| ${row.scenario} | ${cell(s.wrCps, b.wrCps)} | ${cell(s.rdCps, b.rdCps)} | ${cell(s.wrKiBps, b.wrKiBps)} | ${cell(s.rdKiBps, b.rdKiBps)} | ${cell(s.firstMs, b.firstMs)} | ${cell(row.p75, row.baselineP75)} | ${cell(row.p90, row.baselineP90)} | ${cell(row.p99, row.baselineP99)} | ${cell(s.cdvMaxMs, b.cdvMaxMs)} | ${s.iterations} |` + ); + } + return lines.join('\n'); +} + function renderResultTable(result) { const lines = [ '| Metric | Scenario | Best (ms) | P75 (ms) | P90 (ms) | P99 (ms) | Samples |', '|--------|----------|----------:|---------:|---------:|---------:|--------:|', ]; - const rows = [...result.metrics].sort( - (a, b) => metricSortKey(a) - metricSortKey(b) - ); + // Drill-down rows (e.g. CRTT's per-bucket splits) and stream rows (their + // own table) stay out of the headline table. + const rows = result.metrics + .filter((row) => !row.detail && !row.stream) + .sort((a, b) => metricSortKey(a) - metricSortKey(b) || pairedSortKey(a, b)); + if (rows.length === 0) return ''; for (const row of rows) { const label = METRIC_LABELS[row.metric]; // Abbreviations only — the definitions live in the comment footer. @@ -586,13 +866,18 @@ function renderEntry(entry, { heading }) { } else { lines.push(`Backend: \`${result.backend}\` · app: \`${result.app}\``, ''); } - lines.push(renderResultTable(result), ''); - // Only the latest entry carries raw samples (they are stripped before - // being embedded in the comment's data block, see stripRawSamples), so - // this renders for the current run and is silently skipped for the - // collapsed history entries. + const resultTable = renderResultTable(result); + if (resultTable) lines.push(resultTable, ''); + const streamTable = renderStreamTable(result); + if (streamTable) lines.push(streamTable, ''); + // Only the latest entry carries raw samples and histograms (they are + // stripped before being embedded in the comment's data block, see + // stripRawSamples), so these render for the current run and are silently + // skipped for the collapsed history entries. const stsoDiff = renderStsoDiffSection(result); if (stsoDiff) lines.push(stsoDiff, ''); + const crttDiff = renderCrttMatrixSection(result); + if (crttDiff) lines.push(crttDiff, ''); } return lines.join('\n'); } @@ -610,6 +895,25 @@ function buildScenarioLegend(results) { .join(' · '); } +/** + * Replay-cadence identity line: capture id + full semantic sha256 (the + * cross-system workload fingerprint — durabench computes the same hash over + * its copy of the capture; see cadenceSemanticSha256 in benchmark.test.ts). + * Rendered as its own line so the full hash is findable and copyable rather + * than buried in the scenario prose. + */ +function buildCadencesLegend(results) { + const cadences = new Map(); + for (const result of results) { + for (const c of result.config?.replayCadences ?? []) { + if (c?.id && c?.semanticSha256 && !cadences.has(c.id)) { + cadences.set(c.id, c.semanticSha256); + } + } + } + return [...cadences].map(([id, sha]) => `**${id}** \`${sha}\``).join(' · '); +} + /** Targets legend, derived from the per-row targets in the results. */ function buildTargetsLegend(results) { const targets = new Map(); @@ -630,10 +934,35 @@ function buildTargetsLegend(results) { function renderFooter(entries) { const results = entries.flatMap((entry) => entry.results ?? []); - const definitions = METRIC_ORDER.map( - (id) => `**${METRIC_LABELS[id].name}**: ${METRIC_LABELS[id].description}` - ).join(' · '); + // Only define the metrics this comment actually shows — retired metrics + // (e.g. SL/SO, superseded by CRTT) stay defined in METRIC_LABELS so older + // history entries keep rendering, but they drop out of the legend once the + // latest run no longer reports them. + // Only rendered rows feed the legend — artifact-only detail rows (e.g. + // per-index CRTT splits, slip tails) don't define terms the comment never + // shows. + const presentMetrics = new Set( + results.flatMap((result) => + (result.metrics ?? []) + .filter((row) => !row.detail) + .map((row) => row.metric) + ) + ); + // The stream table's columns are CRTT percentiles and CDV max, so those + // definitions stay in the legend whenever stream rows render even though + // no row carries those metric ids anymore. + if (results.some((result) => (result.metrics ?? []).some((r) => r.stream))) { + presentMetrics.add('crtt'); + presentMetrics.add('cdv'); + presentMetrics.delete('stream'); + } + const definitions = METRIC_ORDER.filter((id) => presentMetrics.has(id)) + .map( + (id) => `**${METRIC_LABELS[id].name}**: ${METRIC_LABELS[id].description}` + ) + .join(' · '); const scenarioLegend = buildScenarioLegend(results); + const cadencesLegend = buildCadencesLegend(results); const targetsLegend = buildTargetsLegend(results); const hasBaseline = results.some((result) => (result.metrics ?? []).some( @@ -651,10 +980,32 @@ function renderFooter(entries) { ) ); + const hasCrttDistribution = results.some((result) => + (result.metrics ?? []).some( + (row) => row.stream && Array.isArray(row.hist?.counts) + ) + ); + + const hasStreamTable = results.some((result) => + (result.metrics ?? []).some((row) => row.stream) + ); + const smallprint = [ + ...(hasStreamTable + ? [ + '**Streams**: writer/reader sustained rates (steady window, 10% trimmed each side), first-chunk RTT (the stream-open path, before any buffering/backpressure), CRTT percentiles, and worst delivery stall (CDV max). Cells are medians across iterations; per-run values in the artifacts. No \ud83d\udd34/\ud83d\udfe2 marks until targets attach.', + '', + ] + : []), ...(hasStsoDistribution ? [ - 'The collapsed **STSO distribution** section above buckets every step gap of the sequential-steps run (not a sampled window), split by whether the step ending the gap ran **inline** — in the same warm process as the step before it, so the gap is pure framework overhead — or after a **queue-hop** — the first step of a fresh process, which pays queue dispatch, client reinit and event-log replay. Bars overlay the two runs: `█` is `main`, `┃` marks where this run lands, `░` bridges the gap when this run has more samples in a bucket.', + 'The collapsed **STSO distribution** section above buckets every step gap, split **inline** (same warm process — pure framework overhead) vs **queue-hop** (fresh process — dispatch, reinit, replay). `█` = `main`, `┃` = this run, `░` = fill.', + '', + ] + : []), + ...(hasCrttDistribution + ? [ + 'The collapsed **CRTT drill-down**: per-variant RTT histograms (fixed log bins, `·` = empty) and mean RTT/positive-CDV profile lines over stream progress and chunk size. Histograms, avgs, and profiles merge exactly across runs; p50–p99 are percentile-of-percentiles. Per-index rows live in the artifacts.', '', ] : []), @@ -666,6 +1017,9 @@ function renderFooter(entries) { : []), `Metrics — ${definitions}`, ...(scenarioLegend ? ['', `Scenarios — ${scenarioLegend}`] : []), + ...(cadencesLegend + ? ['', `Replay cadences (semantic sha256) — ${cadencesLegend}`] + : []), ...(targetsLegend ? [ '', @@ -673,9 +1027,9 @@ function renderFooter(entries) { ] : []), '', - 'All metrics are measured from deployment-side timestamps only. Runs are triggered by an in-deployment route that stamps the anchor (`clientStart`) right before `start()`, so the CI runner’s request and its path through api.vercel.com sit outside every measured window. TTFS = in-deployment `start()` → first step body (turbo uses the in-process fast path, non-turbo the dispatch path), and includes the VQS dispatch hop plus any `/flow` cold start. Fan-out TTFS/TTLS are the first and last step completions of a single `Promise.all` over trivial steps, from the same anchor, so the gap between the two rows is the spread the runtime adds across the fan-out. STSO/WO are measured between step bodies on the deployment. SL is measured inside the workflow (parallel reader/writer steps), so it no longer includes the api.vercel.com read path.', + 'All timestamps are deployment-side; runs are triggered in-deployment, so the CI runner and api.vercel.com sit outside every measured window. TTFS = `start()` → first step body (includes dispatch + any cold start); Fan-out TTFS/TTLS = first/last step completion of one `Promise.all` from the same anchor (the gap is the runtime’s fan-out spread); STSO/WO between step bodies; CRTT inside the workflow (excludes the api.vercel.com read path).', '', - 'Cold starts are kept in the numbers on purpose — they are part of real bursty-workload latency. The workbench deployment cold-starts the `/flow` invocation for a large fraction of runs, inflating P75+; the **Best** column shows the fastest (warm-start) sample for comparison.', + 'Cold starts stay in the numbers (real bursty-workload latency, inflates P75+); **Best** is the warm floor.', ]; // Keep the definitions/methodology out of the way in a collapsed dropdown, diff --git a/.github/scripts/render-benchmark-comment.test.js b/.github/scripts/render-benchmark-comment.test.js index b403b51c76..c00a4323ed 100644 --- a/.github/scripts/render-benchmark-comment.test.js +++ b/.github/scripts/render-benchmark-comment.test.js @@ -23,6 +23,17 @@ function sampleResult(overrides = {}) { sequentialIterations: 1, sequentialStepCount: 1020, warmupIterations: 2, + replayCadences: [ + { + id: 'eve-test-cadence', + model: 'test-model', + events: 823, + spanMs: 6196, + totalBytes: 2000000, + semanticSha256: + '609bc99fb5eb810086dcaecc9128f5fecd7c75d8bc3f2b39a6622f89d5a5a47a', + }, + ], }, scenarios: [ { name: 'stream', description: 'one streaming step in turbo mode' }, @@ -120,6 +131,11 @@ test('renders a completed run with a table and embedded history', async () => { body, /Scenarios — \*\*stream\*\*: one streaming step in turbo mode/ ); + // Replay-cadence identity line: full semantic hash on its own legend line + assert.match( + body, + /Replay cadences \(semantic sha256\) — \*\*eve-test-cadence\*\* `609bc99fb5eb810086dcaecc9128f5fecd7c75d8bc3f2b39a6622f89d5a5a47a`<\/sub>/ + ); // Target marks: TTFS p75 398 > 200 → 🔴; SL row is within target on every // percentile, so it stays unmarked (no 🟢 anywhere); WO has no targets. assert.match(body, /398 🔴/); @@ -580,6 +596,210 @@ function sequentialResult({ inline, queueHop }) { }); } +// Fixed log-bin edges matching RTT_HIST_EDGES_MS in the bench helper module +// (workbench/example/workflows/97_bench_rtt.ts). +const CRTT_EDGES = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000]; + +/** Histogram over CRTT_EDGES with counts placed by (value, count) pairs. */ +function crttHist(entries) { + const counts = new Array(CRTT_EDGES.length + 1).fill(0); + for (const [value, count] of entries) { + let bin = 0; + while (bin < CRTT_EDGES.length && value >= CRTT_EDGES[bin]) bin++; + counts[bin] += count; + } + return counts; +} + +function crttResult({ avg = 120, hist }) { + const streamRow = (scenario, group, extra = {}) => ({ + metric: 'stream', + scenario, + unit: 'ms', + best: 59, + avg, + p50: 128, + p75: 188, + p90: 438, + p99: 1229, + samples: hist.reduce((a, b) => a + b, 0), + raw: [], + hist: { edgesMs: CRTT_EDGES, counts: hist }, + group, + bucket: 'all', + stream: { + iterations: 10, + wrCps: 100, + wrKiBps: 6.1, + rdCps: 99.4, + rdKiBps: 6, + firstMs: 96, + cdvMaxMs: 141, + runs: [ + { wrCps: 100, rdCps: 99.4, firstMs: 96, cdvMaxMs: 141, slipMaxMs: 4 }, + ], + }, + ...extra, + }); + return sampleResult({ + scenarios: [ + { name: 'chunk RTT (llm)', description: 'self-timestamping chunks' }, + ], + metrics: [ + streamRow('chunk RTT (llm)', 'llm', { + progressAvgMs: [110, 112, 115, 113, 118, 120, 119, 125, 130, 135], + cdvAvgMs: [2, 2, 3, 5, 9, 15, 24, 40, 66, 108], + }), + // Artifact-only detail rows: per-index CRTT split and slip tail. + { + metric: 'crtt', + scenario: 'chunk RTT llm (seq 0)', + unit: 'ms', + best: 97, + avg: 130, + p50: 112, + p75: 126, + p90: 129, + p99: 157, + samples: 10, + raw: [], + group: 'llm', + bucket: 'seq 0', + detail: true, + }, + { + metric: 'slip', + scenario: 'write slip (llm)', + unit: 'ms', + best: 2, + avg: 3, + p50: 3, + p75: 4, + p90: 5, + p99: 6, + samples: 10, + raw: [], + group: 'llm', + detail: true, + }, + streamRow('replay eve-test (2x)', 'replay', { + stream: { + iterations: 5, + wrCps: 297, + wrKiBps: 742, + rdCps: 288, + rdKiBps: 719, + firstMs: 118, + cdvMaxMs: 210, + runs: [ + { + wrCps: 297, + rdCps: 288, + firstMs: 118, + cdvMaxMs: 210, + slipMaxMs: 9, + }, + ], + }, + }), + ], + }); +} + +test('renders stream scenarios in their own table with rate columns', async () => { + const { renderComment, extractHistory } = await loadModule(); + const hist = crttHist([ + [59, 1400], + [128, 1500], + [438, 100], + ]); + const baseline = crttResult({ avg: 150, hist }); + // Baseline medians differ so deltas render: rd rate was lower on main. + baseline.metrics[0].stream.rdCps = 90; + const body = renderComment({ + status: 'completed', + results: [crttResult({ avg: 120, hist })], + baseline: [baseline], + history: [], + commit: 'abcdef1234567890', + }); + + // Stream rows are OUT of the metric table and IN the Streams table. + assert.doesNotMatch(body, /\| \*\*stream\*\* \|/); + assert.match( + body, + /\| Scenario \| wr c\/s \| rd c\/s \| wr KiB\/s \| rd KiB\/s \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/ + ); + // Rate cells with plain vs-main deltas, latency cells from percentile + // baselines, and NO red/green marks anywhere in the stream table. + assert.match( + body, + /\| chunk RTT \(llm\) \| 100 \(\u00b10%\) \| 99\.4 \(\+10%\) \| 6\.1 \(\u00b10%\) \|/ + ); + assert.match( + body, + /\| replay eve-test \(2x\) \| 297 \(\u00b10%\) \| 288 \(\u00b10%\) \| 742 \(\u00b10%\) \| 719 \(\u00b10%\) \|/ + ); + assert.match(body, /\| 141 \(\u00b10%\) \| 10 \|/); + const streamsSection = body.slice( + body.indexOf('**Streams**'), + body.indexOf('') + ); + assert.doesNotMatch( + streamsSection, + /\ud83d\udd34|\ud83d\udfe2|\ud83d\udd3b|\ud83d\udc9a/ + ); + // Detail rows render nowhere. + assert.doesNotMatch(body, /seq 0 \|/); + assert.doesNotMatch(body, /write slip/); + // Drill-down still renders from the stream rows. + assert.match(body, /\ud83d\udcc8 CRTT drill-down/); + assert.match( + body, + /llm +\u00b7+[\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588]*\u2588/ + ); + assert.match(body, /Delivery jitter over stream progress/); + // CRTT + CDV definitions stay in the legend (stream table columns), and + // the internal 'stream' id never leaks into it. + assert.match(body, /\*\*CRTT\*\*: chunk round-trip time/); + assert.match(body, /\*\*CDV\*\*: chunk delay variation/); + assert.match(body, /\*\*Streams\*\*: writer\/reader sustained rates/); + // History block: per-run arrays and sparkline payloads stripped, medians + // and baseline annotations kept. + const history = extractHistory(body); + const kept = history[0].results[0].metrics[0]; + assert.strictEqual(kept.hist, undefined); + assert.strictEqual(kept.progressAvgMs, undefined); + assert.strictEqual(kept.stream.runs, undefined); + assert.strictEqual(kept.stream.wrCps, 100); + assert.strictEqual(kept.baselineStream.rdCps, 90); + // Re-render from history keeps the Streams table, drops the drill-down. + const rerendered = renderComment({ + status: 'running', + results: [], + history, + commit: 'ffffff1234567890', + }); + assert.match(rerendered, /\| chunk RTT \(llm\) \| 100/); + assert.doesNotMatch(rerendered, /CRTT drill-down/); +}); + +test('renders the stream table without deltas when main has no baseline', async () => { + const { renderComment } = await loadModule(); + const body = renderComment({ + status: 'completed', + results: [crttResult({ hist: crttHist([[128, 3000]]) })], + history: [], + commit: 'abcdef1234567890', + }); + assert.match( + body, + /\| chunk RTT \(llm\) \| 100 \| 99\.4 \| 6\.1 \| 6 \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/ + ); + assert.doesNotMatch(body, /%\)/); + assert.match(body, /No `main` baseline yet/); +}); + test('renders inline and queue-hop STSO histogram diffs against main', async () => { const { renderComment } = await loadModule(); const body = renderComment({ diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 370af08e22..4f5cfe83b2 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,8 +1,8 @@ name: Performance Benchmarks -# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, SL — -# see packages/core/e2e/benchmark.test.ts for definitions) against a deployed -# workbench app and posts the results as a sticky PR comment. Re-runs update +# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, CRTT, +# CDV — see packages/core/e2e/benchmark.test.ts for definitions) against a +# deployed workbench app and posts the results as a sticky PR comment. Re-runs update # the same comment; previous results stay available in a collapsed history # section (state is embedded in the comment body itself). # @@ -96,10 +96,10 @@ jobs: # backend (e.g. postgres or local), add a matrix entry with # `world: postgres` / `world: local` and gate the Vercel-specific steps — # the runner (packages/core/e2e/benchmark.test.ts) already selects its - # backend from the same env vars as the e2e tests. SL is measured inside the - # workflow (benchSlWorkflow's parallel reader/writer steps read/write on the - # deployment), so it no longer needs `run.getReadable()` to work from the - # test process; the runner only polls returnValue for the collected timings. + # backend from the same env vars as the e2e tests. CRTT is measured inside + # the workflow (benchCrttWorkflow's parallel reader/writer steps read/write + # on the deployment), so it does not need `run.getReadable()` to work from + # the test process; the runner only polls returnValue for the aggregates. benchmark: name: Benchmark (${{ matrix.target.world }}, ${{ matrix.target.app }}) runs-on: ubuntu-latest diff --git a/packages/core/e2e/benchmark.test.ts b/packages/core/e2e/benchmark.test.ts index a64d95e717..7b59719cc0 100644 --- a/packages/core/e2e/benchmark.test.ts +++ b/packages/core/e2e/benchmark.test.ts @@ -53,24 +53,36 @@ * `(lastStep.end - clientStart) - Σ(step durations)`. Measured on the * sequential scenario only — on a single-step workflow WO reduces * algebraically to TTFS. - * - SL (stream latency): live write->read propagation for the default - * output stream, measured entirely on the deployment by - * `benchSlWorkflow`: a reader step and a writer step run in parallel, - * the reader blocks on the first chunk, and the workflow returns both - * the writer's `writtenAt` and the reader's `readAt`. SL is - * `readAt - writtenAt`, so it excludes the api.vercel.com read path - * the old client-observed metric included. - * - SO (stream overhead): end-to-end write+consume time in excess of a - * modelled generation window, measured on the deployment by - * `benchSoWorkflow`. A writer streams deterministic variable-length - * LLM-token deltas at a fixed rate for a fixed duration while a - * parallel reader drains the whole stream; SO is - * `(doneAt - writtenAt) - chunkCount*intervalMs`, i.e. the - * overhead/backpressure the stream adds on top of the token rate. Same - * setup as SL, but the reader stamps `doneAt` after the last chunk - * rather than `readAt` on the first. Measured for two payload shapes - * (raw text vs AI-SDK-style structured deltas) so the SO delta between - * them isolates serialization cost. + * - CRTT (chunk round-trip time): per-chunk write->read latency, measured + * on the deployment by benchCrttWorkflow. The "round trip" is + * deployment -> stream backend -> reader on the SAME deployment + * (one clock domain), not an echo to the writer. NAMING: CRTT is + * reserved for this same-clock measurement; the future production + * cross-clock one-way metric is CTT. Every delta embeds + * { seq, writtenAt }; writer and reader run in parallel behind a + * reader-ready barrier (chunk 0 is a live delivery). CRTT subsumes + * the retired SL/SO rows: SL = the seq-0 slice, SO = last-chunk RTT + * + stall accumulation, at ~100x the samples. Aggregation happens + * INSIDE the reader step: index buckets (seq 0 / 1-20 / 21+), fixed + * log-bin histograms, and mean-RTT profiles over stream progress + * and chunk size; the runner merges per-iteration summaries (exact + * best/avg/count/hist, percentile-of-percentiles for p50-p99). + * Per-index rows are artifact-only (detail: true). No targets yet. + * - CDV (chunk delay variation, "delivery jitter"): for seq-adjacent + * chunks received back to back, cdv_i = CTT_i - CTT_{i-1}, computed + * from RAW unclamped timestamps. Each gap subtracts same-clock + * stamps, so CDV is skew-free — the one per-chunk stat measurable + * in production across clock domains. Positives are clumps/stalls, + * negatives catch-up, means telescope away — the sample unit is + * each run's MAX positive cdv. Writer pauses self-exclude, which is + * why write slip (writtenAt - scheduledAt vs the open-loop absolute + * schedule) stays as artifact-only data: it is the producer-stall + * guard neither CRTT nor CDV can see. + * - STREAM TABLE: stream scenarios render as one row each in their own + * table: writer/reader sustained rates (steady window, 10% trimmed + * each side), first-chunk RTT (seq-0, the retired SL signal), CRTT + * p75/p90/p99, CDV max. Cells are medians of per-run values (kept + * in the artifacts). No 🔴/🟢 marks until targets attach. * * Scenarios (defined in workbench/example/workflows/97_bench.ts): * @@ -80,18 +92,22 @@ * 4. benchSequentialStepsWorkflow — 1020 trivial sequential steps → STSO + WO * 5. benchFanOutStepsWorkflow — Promise.all over 100 trivial steps * → Fan-out TTFS + Fan-out TTLS - * 6. benchSlWorkflow — parallel reader/writer steps → SL - * 7. benchSoWorkflow — paced LLM-shaped stream, drained → SO - * (run in text and structured payload modes) + * 6. benchCrttWorkflow — paced stream of self-timestamping chunks → + * CRTT/CDV/rates (llm-shaped and size-sweep + * variants) + * 7. benchReplayWorkflow — replays REAL captured stream cadences + * (write instants + chunk sizes from the + * capture, speed multiplier the only knob) + * → replay rows * * Each scenario runs many iterations (env-tunable, see BENCH_* below) so the * percentiles are computed from real samples. * * The backend is selected exactly like the e2e tests (setupWorld): Vercel when * WORKFLOW_VERCEL_ENV is set, Postgres when WORKFLOW_TARGET_WORLD is - * @workflow/world-postgres, local filesystem otherwise. Because SL is now - * measured inside the workflow (not by a reader in this process), it no longer - * depends on `run.getReadable()` working across processes; CI still runs this + * @workflow/world-postgres, local filesystem otherwise. CRTT is measured + * inside the workflow (not by a reader in this process), so it does not + * depend on `run.getReadable()` working across processes; CI still runs this * file against Vercel only. * * All timestamps are deployment-side, so the only residual skew is intra-Vercel @@ -99,10 +115,22 @@ * relative to the measured values. */ +import { createHash } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { afterAll, beforeAll, describe, test } from 'vitest'; import { getTrustedSourcesHeaders } from '../../../scripts/trusted-sources-headers.mjs'; +import { BENCH_CADENCES } from '../../../workbench/example/workflows/97_bench_cadence'; +import { + type BenchDelayTail, + type BenchRttMeanProfile, + type BenchRttSummary, + type BenchSteadyRate, + mergeMeanProfiles, + mergeRttSummaries, + RTT_HIST_EDGES_MS, + RTT_INDEX_BUCKETS, +} from '../../../workbench/example/workflows/97_bench_rtt'; import { getRun } from '../src/runtime'; import { setupWorld } from './utils'; @@ -123,12 +151,13 @@ const envInt = (name: string, fallback: number, min = 1): number => { return value; }; -// Iteration counts. The stream/hook/SL scenarios yield one sample per +// Iteration counts. The stream/hook scenarios yield one sample per // iteration; the sequential scenario yields (stepCount - 1) STSO samples per // iteration, so a single long run already provides solid percentiles. const STREAM_ITERATIONS = envInt('BENCH_STREAM_ITERATIONS', 30); -const SL_ITERATIONS = envInt('BENCH_SL_ITERATIONS', STREAM_ITERATIONS); -const SO_ITERATIONS = envInt('BENCH_SO_ITERATIONS', STREAM_ITERATIONS); +// Each CRTT iteration yields one RTT sample per chunk (300 by default), so +// few iterations already give thousands of samples per bucket. +const CRTT_ITERATIONS = envInt('BENCH_CRTT_ITERATIONS', 10); const SEQUENTIAL_ITERATIONS = envInt('BENCH_SEQUENTIAL_ITERATIONS', 1); const SEQUENTIAL_STEP_COUNT = envInt('BENCH_SEQUENTIAL_STEP_COUNT', 1020); // The fan-out scenario yields exactly one TTFS and one TTLS sample per @@ -151,21 +180,47 @@ const BENCH_METHODOLOGY_VERSION = 2; // Provisional: now that the proxy leg is out of every window, these will be // re-tightened once a few in-deployment baselines land. const TTFS_TARGETS = { p75: 200, p90: 300, p99: 600 }; -const SL_TARGETS = { p75: 50, p90: 60, p99: 125 }; -// SO scenario: model a haiku-size LLM streaming tokens — ~100 tokens/sec, each -// token a 4-byte chunk, for 3 seconds (300 chunks). The writer paces itself so -// the write phase spans exactly `SO_CHUNK_COUNT * SO_INTERVAL_MS` ms; SO is the -// end-to-end write+consume time beyond that window (see runSoIteration). These -// derive `SO_NOMINAL_DURATION_MS`, the single value subtracted from the -// measured span, so the workflow's write span and the subtraction never drift. -const SO_CHUNK_RATE_PER_SEC = envInt('BENCH_SO_CHUNK_RATE', 100); -const SO_DURATION_SECONDS = envInt('BENCH_SO_DURATION_SECONDS', 3); -const SO_CHUNK_COUNT = SO_CHUNK_RATE_PER_SEC * SO_DURATION_SECONDS; -const SO_INTERVAL_MS = 1000 / SO_CHUNK_RATE_PER_SEC; -const SO_NOMINAL_DURATION_MS = SO_CHUNK_COUNT * SO_INTERVAL_MS; -// Provisional, like TTFS/SL above: re-tighten once in-deployment baselines land. -const SO_TARGETS = { p75: 250, p90: 500, p99: 1000 }; +// CRTT workload: model a haiku-size LLM streaming tokens — ~100 tokens/sec +// for 3 seconds (300 chunks). The writer paces itself so the write phase +// spans exactly `CRTT_CHUNK_COUNT * CRTT_INTERVAL_MS` ms. +const CRTT_CHUNK_RATE_PER_SEC = envInt('BENCH_CRTT_CHUNK_RATE', 100); +const CRTT_DURATION_SECONDS = envInt('BENCH_CRTT_DURATION_SECONDS', 3); +const CRTT_CHUNK_COUNT = CRTT_CHUNK_RATE_PER_SEC * CRTT_DURATION_SECONDS; +const CRTT_INTERVAL_MS = 1000 / CRTT_CHUNK_RATE_PER_SEC; + +// Replay workload: REAL captured cadences (provenance in +// 97_bench_cadence.ts) — every write instant and chunk size comes from a +// capture, one per boundary (eve = demanding envelope protocol, gateway = +// typical raw SSE). The speed multiplier is the only chosen parameter; 2x +// matches how real fast-tier models behave (same chunk sizes, compressed +// time) and exceeds every fast tier measured. +const REPLAY_SPEED = envInt('BENCH_REPLAY_SPEED', 2); +const REPLAY_CADENCE_EVE = 'eve-gpt-5.6-sol-2000t'; // 2593 ev / 52.4s / 16.4MiB +const REPLAY_CADENCE_GATEWAY = 'gateway-gpt-5.4-nano-2000t'; // 1765 ev / 19.9s / 322KiB +// Eve replays cost ~26s (2x) / ~52s (1x) wall per iteration — few +// iterations there, more on the cheap gateway row. 1x = reality (not +// implied by a strained 2x row, and the more linear regression detector); +// 2x = headroom. +/** + * Cross-system cadence identity: durabench carries its own copy of each + * capture, so both sides hash canonical event tuples (format-independent). + * CANONICAL FORM (keep in sync with durabench): sha256 over UTF-8 + * "v1\n" + ",\n" per event, base-10, LF separators. + */ +function cadenceSemanticSha256(cadenceId: string): string { + const cadence = BENCH_CADENCES[cadenceId]; + const hash = createHash('sha256'); + hash.update('v1\n'); + for (let i = 0; i < cadence.offsetsMs.length; i++) { + hash.update(`${cadence.offsetsMs[i]},${cadence.sizes[i]}\n`); + } + return hash.digest('hex'); +} + +const REPLAY_EVE_ITERATIONS = envInt('BENCH_REPLAY_EVE_ITERATIONS', 3); +const REPLAY_REALITY_ITERATIONS = envInt('BENCH_REPLAY_REALITY_ITERATIONS', 2); +const REPLAY_GATEWAY_ITERATIONS = envInt('BENCH_REPLAY_GATEWAY_ITERATIONS', 3); // Guard timeouts so a single stuck run fails fast instead of eating the job. const RUN_TIMEOUT_MS = envInt('BENCH_RUN_TIMEOUT_MS', 120_000); @@ -195,17 +250,6 @@ interface BenchStepTiming { kind: 'inline' | 'queue-hop'; } -interface BenchStreamLatency { - writtenAt: number; - readAt: number; -} - -interface BenchStreamOverhead { - writtenAt: number; - doneAt: number; - received: number; -} - interface StreamIterationResult { runId: string; /** `steps[0].start - clientStart`, both deployment-side clocks. */ @@ -240,16 +284,34 @@ interface FanOutIterationResult { fanOutTtlsMs: number; } -interface SlIterationResult { - runId: string; - /** `readAt - writtenAt`, both deployment-side step-body clocks. */ - slMs: number; +/** Mirrors BenchChunkRttResult in workflows/97_bench.ts: per-bucket RTT + * summaries aggregated inside the reader step (buckets without samples are + * absent). */ +interface BenchChunkRttResult { + received: number; + all?: BenchRttSummary; + byIndex: Partial>; + progress?: BenchRttMeanProfile; + size?: BenchRttMeanProfile; + cdv?: BenchChunkCdv; + delivered?: BenchSteadyRate; } -interface SoIterationResult { +/** Mirrors BenchChunkCdv in workflows/97_bench.ts. */ +interface BenchChunkCdv { + pairs: number; + skippedPairs: number; + positive?: BenchDelayTail; + progress?: BenchRttMeanProfile; +} + +interface CrttIterationResult { runId: string; - /** `(doneAt - writtenAt) - SO_NOMINAL_DURATION_MS`, deployment-side clocks. */ - soMs: number; + crtt: BenchChunkRttResult; + /** Writer-side pacing slip for the run (artifact-only guard). */ + writeSlip?: BenchDelayTail; + /** Writer-side achieved sustained rate over the steady window. */ + achieved?: BenchSteadyRate; } /** Response shape of the in-deployment `POST /api/bench` trigger route. */ @@ -464,74 +526,129 @@ async function runFanOutIteration( } } -async function runSlIteration(): Promise { - const { runId } = await triggerBenchRun('benchSlWorkflow'); +async function runCrttIteration( + variant: 'llm' | 'sweep', + chunkCount: number, + intervalMs: number +): Promise { + const { runId } = await triggerBenchRun('benchCrttWorkflow', [ + chunkCount, + intervalMs, + variant, + ]); try { const returnValue = await withTimeout( getReturnValue(runId), - RUN_TIMEOUT_MS, - `benchSlWorkflow returnValue (run ${runId})` + // The writer streams for the whole generation window before the run can + // complete, so extend the guard past the base run timeout by that window. + RUN_TIMEOUT_MS + chunkCount * intervalMs, + `benchCrttWorkflow (${variant}) returnValue (run ${runId})` ); - const sl = (returnValue as { sl?: BenchStreamLatency } | undefined)?.sl; - if ( - !sl || - typeof sl.writtenAt !== 'number' || - typeof sl.readAt !== 'number' - ) { + const { crtt, writeSlip, achieved } = + (returnValue as + | { + crtt?: BenchChunkRttResult; + writeSlip?: BenchDelayTail; + achieved?: BenchSteadyRate; + } + | undefined) ?? {}; + if (!crtt || !crtt.all || typeof crtt.all.avg !== 'number') { throw new Error( - `Run ${runId} returned no stream-latency sample: ${JSON.stringify(returnValue)?.slice(0, 200)}` + `Run ${runId} returned no chunk-RTT summaries: ${JSON.stringify(returnValue)?.slice(0, 200)}` ); } - return { runId, slMs: Math.max(0, sl.readAt - sl.writtenAt) }; + if (crtt.received !== chunkCount) { + throw new Error( + `Run ${runId} consumed ${crtt.received} chunks, expected ${chunkCount}` + ); + } + return { runId, crtt, writeSlip, achieved }; } catch (error) { (error as Error).message += ` (run ${runId})`; throw error; } } -async function runSoIteration( - mode: 'text' | 'structured' -): Promise { - const { runId } = await triggerBenchRun('benchSoWorkflow', [ - SO_CHUNK_COUNT, - SO_INTERVAL_MS, - mode, +async function runReplayIteration( + cadenceId: string, + speed: number +): Promise { + const cadence = BENCH_CADENCES[cadenceId]; + const { runId } = await triggerBenchRun('benchReplayWorkflow', [ + cadenceId, + speed, ]); try { const returnValue = await withTimeout( getReturnValue(runId), - // The writer streams for the whole generation window before the run can - // complete, so extend the guard past the base run timeout by that window. - RUN_TIMEOUT_MS + SO_NOMINAL_DURATION_MS, - `benchSoWorkflow (${mode}) returnValue (run ${runId})` + RUN_TIMEOUT_MS + cadence.spanMs / speed, + `benchReplayWorkflow (${cadenceId} ${speed}x) returnValue (run ${runId})` ); - const so = (returnValue as { so?: BenchStreamOverhead } | undefined)?.so; - if ( - !so || - typeof so.writtenAt !== 'number' || - typeof so.doneAt !== 'number' - ) { + const { crtt, writeSlip, achieved } = + (returnValue as + | { + crtt?: BenchChunkRttResult; + writeSlip?: BenchDelayTail; + achieved?: BenchSteadyRate; + } + | undefined) ?? {}; + if (!crtt || !crtt.all || typeof crtt.all.avg !== 'number') { throw new Error( - `Run ${runId} returned no stream-overhead sample: ${JSON.stringify(returnValue)?.slice(0, 200)}` + `Run ${runId} returned no chunk-RTT summaries: ${JSON.stringify(returnValue)?.slice(0, 200)}` ); } - if (so.received !== SO_CHUNK_COUNT) { + if (crtt.received !== cadence.events) { throw new Error( - `Run ${runId} consumed ${so.received} chunks, expected ${SO_CHUNK_COUNT}` + `Run ${runId} consumed ${crtt.received} chunks, expected ${cadence.events}` ); } - // Both timestamps are deployment-side; subtract the modelled generation - // window and clamp to absorb tiny intra-Vercel skew. - return { - runId, - soMs: Math.max(0, so.doneAt - so.writtenAt - SO_NOMINAL_DURATION_MS), - }; + return { runId, crtt, writeSlip, achieved }; } catch (error) { (error as Error).message += ` (run ${runId})`; throw error; } } +/** + * Median across per-iteration values (undefined skipped); even counts + * average the two middles — lower-middle would report the better of a + * 2-run scenario's runs and call it the median. + */ +function medianOf(values: readonly (number | undefined)[]): number | undefined { + const present = values.filter((v): v is number => typeof v === 'number'); + if (present.length === 0) return undefined; + const sorted = [...present].sort((a, b) => a - b); + const mid = sorted.length / 2; + return sorted.length % 2 === 1 + ? sorted[Math.floor(mid)] + : (sorted[mid - 1] + sorted[mid]) / 2; +} + +/** + * Records the write-slip detail row for a stream variant (artifact-only, + * never rendered). The sample unit is each run's MAX slip: one producer + * stall among thousands of chunks vanishes into a pooled p99 but is, by + * construction, that run's max. Slip is the guard for producer stalls, + * which neither per-chunk RTT (late writes are stamped late) nor CDV + * (writer pauses grow both gaps equally) can see. + */ +function recordSlipDetailRow( + scenario: string, + group: string, + tails: readonly (BenchDelayTail | undefined)[] +) { + const samples = tails.flatMap((tail) => (tail ? [tail.maxMs] : [])); + if (samples.length === 0) return; + metricRows.push({ + metric: 'slip', + scenario, + unit: 'ms', + group, + detail: true, + ...computeStats(samples), + }); +} + /** * Runs recorded iterations (plus warmups) sequentially — concurrency would * contend on the same deployment and skew latencies. Failed iterations are @@ -600,6 +717,9 @@ interface MetricStats { best: number; /** Mean; kept in the JSON for reference but not shown in the PR comment. */ avg: number; + /** Median; only recorded for CRTT rows (the exit criteria track median and + * average per-chunk RTT). Kept in the JSON, not shown in the PR comment. */ + p50?: number; p75: number; p90: number; p99: number; @@ -608,6 +728,56 @@ interface MetricStats { * comment diffs the whole STSO distribution against `main`, and * percentiles alone hide *how many* samples moved and by how much. */ raw: number[]; + /** Fixed-bin histogram of the samples, for rows whose raw samples never + * reach this process (CRTT: aggregation happens in the reader step on the + * deployment). Fixed shared edges make the PR comment's distribution diff + * against `main` exact — the renderer only diffs matching-edge rows. */ + hist?: { edgesMs: number[]; counts: number[] }; + /** Drill-down rows (e.g. CRTT per-bucket splits): kept out of the PR + * comment's main results table and rendered in a collapsed section. */ + detail?: boolean; + /** Mean RTT per tenth of the stream (CRTT headline rows): the drift/trend + * readout, rendered as a progress sparkline in the drill-down. Null + * entries are empty bins (rendered as gaps, never as zero). */ + progressAvgMs?: (number | null)[]; + /** Mean RTT per log size bin (CRTT sweep headline row): the size→latency + * curve, rendered as a size sparkline in the drill-down. Null entries are + * bins the sweep left empty. */ + sizeAvgMs?: (number | null)[]; + /** Mean POSITIVE CDV per tenth of the stream (stream headline rows), + * rendered as a delivery-jitter sparkline in the drill-down: localizes + * where delivery clumping/stalls concentrate. Complements the stream + * table's CDV max column, which says the worst stall's size but not + * where. */ + cdvAvgMs?: (number | null)[]; + /** Stream-scenario columns (marks the row for the PR comment's separate + * stream table): writer/reader sustained rates over the steady window and + * the median worst delivery stall, medians across iterations with the + * per-run values retained in `runs`. */ + stream?: { + iterations: number; + wrCps?: number; + wrKiBps?: number; + rdCps?: number; + rdKiBps?: number; + /** Median across runs of each run's seq-0 RTT — the stream-open path, + * before any buffering/backpressure (the retired SL signal). */ + firstMs?: number; + cdvMaxMs?: number; + runs: { + wrCps?: number; + wrKiBps?: number; + rdCps?: number; + rdKiBps?: number; + firstMs?: number; + cdvMaxMs?: number; + slipMaxMs?: number; + }[]; + }; + /** Short group/bucket labels for drill-down rendering (CRTT: variant and + * index/size bucket). */ + group?: string; + bucket?: string; } interface MetricTargets { @@ -648,6 +818,9 @@ interface MetricRow extends MetricStats { } const metricRows: MetricRow[] = []; +// Per-run seq-0 RTTs from every stream scenario, pooled into the +// 'first chunk (pooled)' main-table row in afterAll. +const firstChunkRttSamples: number[] = []; function recordMetric( metric: string, @@ -665,6 +838,131 @@ function recordMetric( }); } +/** + * Records one CRTT row from per-iteration summaries. Unlike recordMetric + * there are no raw samples in this process — the reader step aggregated them + * on the deployment — so the row is the mergeRttSummaries merge: exact + * count/best/avg/histogram, percentile-of-percentiles for p50-p99. `samples` + * is the total chunk count across iterations. The merged fixed-bin histogram + * rides along for the PR comment's sparkline drill-down (exact vs `main`, + * where the percentiles are approximations). Rows with `detail` are not + * rendered at all — they carry the per-index-bucket splits in the results + * JSON (with baseline annotations) so a headline regression can be localized + * from the artifacts. No targets yet (see the CRTT header note), so no 🔴 + * marks render. + */ +// Mean RTT per profile bin; sums/counts merge exactly across iterations, +// so these avgs are exact like the histogram. Empty bins become null so +// the renderer can show them as gaps rather than zeros. +function profileAvgs(profile?: BenchRttMeanProfile) { + return profile?.totalMs.map((total, i) => + profile.counts[i] > 0 + ? Math.round((total / profile.counts[i]) * 10) / 10 + : null + ); +} + +/** Records an artifact-only per-index-bucket CRTT row (never rendered). */ +function recordCrttDetailRow( + scenario: string, + summaries: readonly (BenchRttSummary | undefined)[], + { group, bucket }: { group: string; bucket: string } +) { + const merged = mergeRttSummaries(summaries); + if (!merged) return; + metricRows.push({ + metric: 'crtt', + scenario, + unit: 'ms', + best: merged.best, + avg: merged.avg, + p50: merged.p50, + p75: merged.p75, + p90: merged.p90, + p99: merged.p99, + samples: merged.count, + raw: [], + hist: { edgesMs: RTT_HIST_EDGES_MS, counts: merged.hist }, + detail: true, + group, + bucket, + }); +} + +/** + * Records one stream-scenario row (headline of the PR comment's STREAM + * table). CRTT percentiles are percentile-of-percentiles across iterations + * (count/best/avg/histograms are exact); rates, first-chunk RTT, and CDV + * max are medians of per-run values, kept per-run in `stream.runs`. + */ +function recordStreamRow( + scenario: string, + group: string, + results: readonly CrttIterationResult[], + { + size, + }: { + /** Include the size→latency profile (sweep variant only). */ + size?: boolean; + } = {} +) { + const merged = mergeRttSummaries(results.map((r) => r.crtt.all)); + if (!merged) return; + for (const r of results) { + const first = r.crtt.byIndex?.['seq 0']?.avg; + if (typeof first === 'number') firstChunkRttSamples.push(first); + } + const runs = results.map((r) => ({ + wrCps: r.achieved?.chunksPerSec, + wrKiBps: r.achieved?.kibPerSec, + rdCps: r.crtt.delivered?.chunksPerSec, + rdKiBps: r.crtt.delivered?.kibPerSec, + // Single sample (the run's seq-0 chunk), so avg IS that run's value. + firstMs: r.crtt.byIndex?.['seq 0']?.avg, + cdvMaxMs: r.crtt.cdv?.positive?.maxMs, + slipMaxMs: r.writeSlip?.maxMs, + })); + metricRows.push({ + metric: 'stream', + scenario, + unit: 'ms', + best: merged.best, + avg: merged.avg, + p50: merged.p50, + p75: merged.p75, + p90: merged.p90, + p99: merged.p99, + samples: merged.count, + raw: [], + hist: { edgesMs: RTT_HIST_EDGES_MS, counts: merged.hist }, + group, + bucket: 'all', + // Nulls (empty bins) are preserved: the renderer draws them as gaps. + // Mapping them to 0 would claim "measured no jitter/latency here" + // rather than "no samples here" — CDV progress bins are legitimately + // empty wherever a tenth of the stream had no positive-cdv chunks. + progressAvgMs: profileAvgs( + mergeMeanProfiles(results.map((r) => r.crtt.progress)) + ), + sizeAvgMs: size + ? profileAvgs(mergeMeanProfiles(results.map((r) => r.crtt.size))) + : undefined, + cdvAvgMs: profileAvgs( + mergeMeanProfiles(results.map((r) => r.crtt.cdv?.progress)) + ), + stream: { + iterations: results.length, + wrCps: medianOf(runs.map((r) => r.wrCps)), + wrKiBps: medianOf(runs.map((r) => r.wrKiBps)), + rdCps: medianOf(runs.map((r) => r.rdCps)), + rdKiBps: medianOf(runs.map((r) => r.rdKiBps)), + firstMs: medianOf(runs.map((r) => r.firstMs)), + cdvMaxMs: medianOf(runs.map((r) => r.cdvMaxMs)), + runs, + }, + }); +} + function getBackend(): string { if (process.env.WORKFLOW_BENCH_BACKEND) { return process.env.WORKFLOW_BENCH_BACKEND; @@ -683,13 +981,22 @@ const SCENARIO_TURBO_STREAM = 'stream'; const SCENARIO_HOOK_STREAM = 'hook + stream'; const SCENARIO_SEQUENTIAL = `${SEQUENTIAL_STEP_COUNT} steps`; const SCENARIO_FANOUT = `Promise.all(${FANOUT_STEP_COUNT} steps)`; -const SCENARIO_STREAM_LATENCY = 'stream latency'; -// Two SO scenarios differing only in payload shape. The labels are distinct -// from the pre-existing 'stream overhead' baseline key, so the payload change -// doesn't diff against the old fixed-'aaaa' numbers — the SO deltas start blank -// and re-baseline on the next `main` run. -const SCENARIO_STREAM_OVERHEAD_TEXT = 'stream overhead (text)'; -const SCENARIO_STREAM_OVERHEAD_STRUCTURED = 'stream overhead (structured)'; +// Stream scenario labels, doubling as the stream-table rows' scenario keys; +// the per-bucket detail rows are keyed `paced control ()` and slip +// detail rows `write slip ()`. All new baseline keys, so the +// stream deltas stay blank until `main` produces them. +// Synthetic rows are named for their role: the metronome is the control +// (diagnostic anchor + flush-cadence probe); the sweep isolates size +// causally (rotation decouples size from position; the replay ramp +// couples them). +const SCENARIO_PACED_CONTROL = 'paced control (100/s, 60B)'; +const SCENARIO_SIZE_SWEEP = 'size sweep (100/s, 160B-12KB)'; +// Capture id + speed IS the baseline key: a re-capture is a new workload +// and starts a new baseline by construction. +// Parenthesized speed, not `@2x`: GitHub renders @ as a user mention. +const SCENARIO_REPLAY_EVE = `replay ${REPLAY_CADENCE_EVE} (${REPLAY_SPEED}x)`; +const SCENARIO_REPLAY_REALITY = `replay ${REPLAY_CADENCE_EVE} (1x)`; +const SCENARIO_REPLAY_GATEWAY = `replay ${REPLAY_CADENCE_GATEWAY} (1x)`; const SCENARIO_DESCRIPTIONS = [ { name: SCENARIO_STEP, @@ -715,19 +1022,34 @@ const SCENARIO_DESCRIPTIONS = [ description: `${FANOUT_STEP_COUNT} trivial no-op steps started together in a single Promise.all; Fan-out TTFS is the first of them to complete and Fan-out TTLS the last, both from the in-deployment clientStart, so their gap is the spread the runtime adds across the fan-out`, }, { - name: SCENARIO_STREAM_LATENCY, - description: - 'parallel reader/writer steps on a dedicated stream; SL is the in-deployment write->read propagation (readAt - writtenAt)', + name: SCENARIO_PACED_CONTROL, + description: `the control: ${CRTT_CHUNK_COUNT} tiny (~60B) deltas metronome-paced at ${CRTT_CHUNK_RATE_PER_SEC}/s — zero workload structure, so it reads the transport floor and flush cadence, and disambiguates transport-wide vs workload-specific when a replay row moves`, + }, + { + name: SCENARIO_SIZE_SWEEP, + description: `same pacing as the control with deltas padded in rotation across seven log-spaced sizes (~160B–12KB) — rotation decouples size from stream position, so it isolates whether chunk size causes latency`, + }, + { + name: SCENARIO_REPLAY_GATEWAY, + description: `raw provider SSE cadence captured at the AI gateway boundary (gpt-5.4-nano, the most popular gateway model; per-token deltas p50 208B = the modal production chunk size), replayed exactly as measured — the typical customer's workload; its CDV is the typical customer's real delivery jitter`, + }, + { + name: SCENARIO_REPLAY_REALITY, + description: `a captured eve turn (gpt-5.6-sol, the most-used demanding eve model; ~2000 output tokens = production p50 turn length) replayed exactly as measured — eve's envelope protocol re-ships the cumulative message so sizes ramp 142B→13KB; the demanding outlier tenant's reality`, }, { - name: SCENARIO_STREAM_OVERHEAD_TEXT, - description: `writer streams ${SO_CHUNK_COUNT} variable-length text token deltas paced at ${SO_CHUNK_RATE_PER_SEC}/s for ${SO_DURATION_SECONDS}s (a haiku-size LLM's token throughput) while a parallel reader drains the whole stream; SO is the end-to-end write+consume time beyond the ${SO_DURATION_SECONDS}s generation window (overhead/backpressure)`, + name: SCENARIO_REPLAY_EVE, + description: `the same eve capture at ${REPLAY_SPEED}x — the headroom/stress row; real fast-tier models emit the same chunk sizes at proportionally higher rate, so time compression is a faithful speed model`, }, { - name: SCENARIO_STREAM_OVERHEAD_STRUCTURED, - description: `same workload as ${SCENARIO_STREAM_OVERHEAD_TEXT}, but each delta is an AI-SDK-style structured object ({ type: 'text-delta', id, text }) instead of a raw string, so the SO gap vs the text scenario is the added serialization cost`, + name: 'first chunk (pooled)', + description: `every run's seq-0 RTT pooled across all stream scenarios — the first chunk precedes any workload differentiation, so pooling samples one shared stream-open path with exact percentiles`, }, ]; +// Cross-system cadence identity: the full semantic hash lands in +// config.replayCadences (rendered as its own "Replay cadences" legend line +// and copyable from the artifacts); durabench computes the same hash over +// its copy (see cadenceSemanticSha256). // Datadog APM permalink for a trace id. The benchmark deployment exports its // OTel spans to Datadog, and `/api/bench` returns the trace id of the request @@ -827,52 +1149,114 @@ describe('workflow benchmarks', () => { } ); - test('scenario: stream latency', { timeout: 30 * 60_000 }, async () => { + test('scenario: paced control', { timeout: 30 * 60_000 }, async () => { const results = await runScenario( - SCENARIO_STREAM_LATENCY, - SL_ITERATIONS, - () => runSlIteration() + SCENARIO_PACED_CONTROL, + CRTT_ITERATIONS, + () => runCrttIteration('llm', CRTT_CHUNK_COUNT, CRTT_INTERVAL_MS) ); - recordMetric( - 'sl', - SCENARIO_STREAM_LATENCY, - results.map((r) => r.slMs), - SL_TARGETS + // One rendered row per stream scenario (in the separate stream table); + // the index-bucket rows split RTT by position in the stream for the + // results artifacts only (flat across runs so far), as do slip tails. + recordStreamRow(SCENARIO_PACED_CONTROL, 'control', results); + for (const bucket of RTT_INDEX_BUCKETS) { + recordCrttDetailRow( + `paced control (${bucket})`, + results.map((r) => r.crtt.byIndex[bucket]), + { group: 'control', bucket } + ); + } + recordSlipDetailRow( + 'write slip (paced control)', + 'control', + results.map((r) => r.writeSlip) ); }); + test('scenario: size sweep', { timeout: 30 * 60_000 }, async () => { + const results = await runScenario( + SCENARIO_SIZE_SWEEP, + CRTT_ITERATIONS, + () => runCrttIteration('sweep', CRTT_CHUNK_COUNT, CRTT_INTERVAL_MS) + ); + // The size→latency profile only makes sense here: the llm-shaped + // deltas all land in the smallest size bin. + recordStreamRow(SCENARIO_SIZE_SWEEP, 'sweep', results, { + size: true, + }); + recordSlipDetailRow( + 'write slip (size sweep)', + 'sweep', + results.map((r) => r.writeSlip) + ); + }); + + // The replay scenarios run (and therefore render) in ascending difficulty: + // gateway 1x (typical customer as measured, the lightest total load) → + // eve 1x (demanding workload as measured) → eve 2x (stress). The gateway + // capture deliberately has no 2x row: nano at 1x already sits at the fast + // end of measured gateway rates, and a first run showed gateway-2x tails + // statistically identical to eve 2x — headroom is eve 2x's job. + test( - 'scenario: stream overhead (text)', + 'scenario: replay (gateway gpt-5.4-nano 2000t, 1x reality)', { timeout: 30 * 60_000 }, async () => { const results = await runScenario( - SCENARIO_STREAM_OVERHEAD_TEXT, - SO_ITERATIONS, - () => runSoIteration('text') + SCENARIO_REPLAY_GATEWAY, + REPLAY_GATEWAY_ITERATIONS, + () => runReplayIteration(REPLAY_CADENCE_GATEWAY, 1), + // ~20s wall per run; one warmup — earlier scenarios already warmed + // the deployment and stream path. + { warmupIterations: 1 } ); - recordMetric( - 'so', - SCENARIO_STREAM_OVERHEAD_TEXT, - results.map((r) => r.soMs), - SO_TARGETS + recordStreamRow(SCENARIO_REPLAY_GATEWAY, 'gw 1x', results); + recordSlipDetailRow( + `write slip (${REPLAY_CADENCE_GATEWAY} 1x)`, + 'gw 1x', + results.map((r) => r.writeSlip) ); } ); test( - 'scenario: stream overhead (structured)', + 'scenario: replay (eve gpt-5.6-sol 2000t, 1x reality)', { timeout: 30 * 60_000 }, async () => { const results = await runScenario( - SCENARIO_STREAM_OVERHEAD_STRUCTURED, - SO_ITERATIONS, - () => runSoIteration('structured') + SCENARIO_REPLAY_REALITY, + REPLAY_REALITY_ITERATIONS, + () => runReplayIteration(REPLAY_CADENCE_EVE, 1), + // ~52s wall per run; no warmup — the gateway 1x replay just ran, so + // the replay path is warm. + { warmupIterations: 0 } ); - recordMetric( - 'so', - SCENARIO_STREAM_OVERHEAD_STRUCTURED, - results.map((r) => r.soMs), - SO_TARGETS + recordStreamRow(SCENARIO_REPLAY_REALITY, 'eve 1x', results); + recordSlipDetailRow( + `write slip (${REPLAY_CADENCE_EVE} 1x)`, + 'eve 1x', + results.map((r) => r.writeSlip) + ); + } + ); + + test( + 'scenario: replay (eve gpt-5.6-sol 2000t, 2x)', + { timeout: 30 * 60_000 }, + async () => { + const results = await runScenario( + SCENARIO_REPLAY_EVE, + REPLAY_EVE_ITERATIONS, + () => runReplayIteration(REPLAY_CADENCE_EVE, REPLAY_SPEED), + // No warmup: the same cadence already replayed at 1x above, so + // everything this touches is warm. + { warmupIterations: 0 } + ); + recordStreamRow(SCENARIO_REPLAY_EVE, 'eve 2x', results); + recordSlipDetailRow( + `write slip (${REPLAY_CADENCE_EVE} ${REPLAY_SPEED}x)`, + 'eve 2x', + results.map((r) => r.writeSlip) ); } ); @@ -970,6 +1354,18 @@ describe('workflow benchmarks', () => { }); afterAll(() => { + // Pooled first-chunk RTTs: seq 0 precedes any workload differentiation + // (no queue depth / backpressure), so every scenario samples one shared + // stream-open path — the one valid cross-scenario pool. ~TTFS-sized + // sample set, exact percentiles (raw samples, no merge). + if (firstChunkRttSamples.length > 0) { + metricRows.push({ + metric: 'crtt', + scenario: 'first chunk (pooled)', + unit: 'ms', + ...computeStats(firstChunkRttSamples), + }); + } if (metricRows.length === 0) { console.warn('[bench] No metrics collected; skipping results file'); return; @@ -992,11 +1388,29 @@ describe('workflow benchmarks', () => { commit: process.env.GITHUB_SHA || undefined, config: { streamIterations: STREAM_ITERATIONS, - slIterations: SL_ITERATIONS, - soIterations: SO_ITERATIONS, - soChunkCount: SO_CHUNK_COUNT, - soChunkRatePerSec: SO_CHUNK_RATE_PER_SEC, - soDurationSeconds: SO_DURATION_SECONDS, + crttIterations: CRTT_ITERATIONS, + crttChunkCount: CRTT_CHUNK_COUNT, + crttChunkRatePerSec: CRTT_CHUNK_RATE_PER_SEC, + crttDurationSeconds: CRTT_DURATION_SECONDS, + replaySpeed: REPLAY_SPEED, + replayEveIterations: REPLAY_EVE_ITERATIONS, + replayRealityIterations: REPLAY_REALITY_ITERATIONS, + replayGatewayIterations: REPLAY_GATEWAY_ITERATIONS, + replayCadences: [REPLAY_CADENCE_EVE, REPLAY_CADENCE_GATEWAY].map( + (id) => { + const c = BENCH_CADENCES[id]; + return { + id, + model: c.model, + capturedAt: c.capturedAt, + eveCommit: c.eveCommit, + events: c.events, + spanMs: c.spanMs, + totalBytes: c.totalBytes, + semanticSha256: cadenceSemanticSha256(id), + }; + } + ), sequentialIterations: SEQUENTIAL_ITERATIONS, sequentialStepCount: SEQUENTIAL_STEP_COUNT, fanoutIterations: FANOUT_ITERATIONS, diff --git a/packages/core/src/bench-chunk-rtt-stats.test.ts b/packages/core/src/bench-chunk-rtt-stats.test.ts new file mode 100644 index 0000000000..6a40509db0 --- /dev/null +++ b/packages/core/src/bench-chunk-rtt-stats.test.ts @@ -0,0 +1,377 @@ +/** + * Unit tests for the chunk-RTT (CRTT) benchmark's pure bucketing/aggregation + * helpers (workbench/example/workflows/97_bench_rtt.ts). The module is + * dependency-free on purpose: the same code runs inside the benchmark's + * reader step on the deployment (per-iteration aggregation) and in the + * benchmark runner (cross-iteration merging), and this suite is the fast + * check on both — the bench itself only runs against a deployment. + */ + +import { describe, expect, test } from 'vitest'; +import { + type BenchRttSummary, + type CdvArrival, + computeCdv, + histogramRttSamples, + mergeMeanProfiles, + mergeRttSummaries, + progressProfile, + RTT_HIST_EDGES_MS, + RTT_INDEX_BUCKETS, + RTT_PROGRESS_BINS, + RTT_SIZE_BIN_EDGES_BYTES, + rttIndexBucket, + rttSizeBin, + sizeProfile, + steadyRate, + summarizeDelayTail, + summarizeRttSamples, +} from '../../../workbench/example/workflows/97_bench_rtt'; + +/** Histogram with `count` in the bin holding `value` and zeros elsewhere. */ +function histWith(value: number, count = 1): number[] { + const hist = new Array(RTT_HIST_EDGES_MS.length + 1).fill(0); + let bin = 0; + while (bin < RTT_HIST_EDGES_MS.length && value >= RTT_HIST_EDGES_MS[bin]) { + bin++; + } + hist[bin] = count; + return hist; +} + +describe('rttIndexBucket', () => { + test('boundaries: stream-open write / warmup / steady state', () => { + expect(rttIndexBucket(0)).toBe('seq 0'); + expect(rttIndexBucket(1)).toBe('seq 1-20'); + expect(rttIndexBucket(20)).toBe('seq 1-20'); + expect(rttIndexBucket(21)).toBe('seq 21+'); + expect(rttIndexBucket(299)).toBe('seq 21+'); + }); + + test('every bucket is a declared bucket key', () => { + for (let seq = 0; seq < 300; seq++) { + expect(RTT_INDEX_BUCKETS).toContain(rttIndexBucket(seq)); + } + }); +}); + +describe('progressProfile', () => { + test('bins by fraction of the stream, so profiles are chunk-count independent', () => { + // 300 chunks: each tenth holds exactly 30. + const rtts = Array.from({ length: 300 }, (_, seq) => seq); + const profile = progressProfile(rtts); + expect(profile.counts).toEqual(new Array(RTT_PROGRESS_BINS).fill(30)); + // First tenth: seq 0..29 (sum 435); last tenth: seq 270..299 (sum 8535). + expect(profile.totalMs[0]).toBe(435); + expect(profile.totalMs[RTT_PROGRESS_BINS - 1]).toBe(8535); + + // 20 chunks (fewer than would fill 10 bins evenly at other counts): still + // 2 per tenth. + const small = progressProfile(Array.from({ length: 20 }, () => 5)); + expect(small.counts).toEqual(new Array(RTT_PROGRESS_BINS).fill(2)); + }); + + test('skips sparse entries defensively', () => { + const rtts: (number | undefined)[] = new Array(100); + rtts[0] = 7; + rtts[99] = 9; + const profile = progressProfile(rtts); + expect(profile.counts.reduce((a, b) => a + b, 0)).toBe(2); + expect(profile.totalMs[0]).toBe(7); + expect(profile.totalMs[RTT_PROGRESS_BINS - 1]).toBe(9); + }); +}); + +describe('mergeMeanProfiles', () => { + test('returns undefined with no profiles and sums exactly otherwise', () => { + expect(mergeMeanProfiles([])).toBeUndefined(); + expect(mergeMeanProfiles([undefined])).toBeUndefined(); + const a = progressProfile(Array.from({ length: 10 }, () => 10)); + const b = progressProfile(Array.from({ length: 10 }, () => 30)); + const merged = mergeMeanProfiles([a, undefined, b]); + expect(merged?.counts).toEqual(new Array(RTT_PROGRESS_BINS).fill(2)); + expect(merged?.totalMs).toEqual(new Array(RTT_PROGRESS_BINS).fill(40)); + }); +}); + +describe('sizeProfile', () => { + test('bins by serialized size with doubling edges', () => { + expect(rttSizeBin(100)).toBe(0); + expect(rttSizeBin(255)).toBe(0); + // A size exactly on an edge lands in the bin the edge opens. + expect(rttSizeBin(256)).toBe(1); + expect(rttSizeBin(1024)).toBe(3); + expect(rttSizeBin(8192)).toBe(RTT_SIZE_BIN_EDGES_BYTES.length); + expect(rttSizeBin(20000)).toBe(RTT_SIZE_BIN_EDGES_BYTES.length); + }); + + test('the sweep pad ladder occupies every size bin exactly once', () => { + // Approximate serialized sizes of the sweep rotation: ~60B base chunk + // plus pads of 100/340/700/1400/3000/6000/12000 chars. + const sizes = [160, 400, 760, 1460, 3060, 6060, 12060]; + expect(new Set(sizes.map(rttSizeBin)).size).toBe( + RTT_SIZE_BIN_EDGES_BYTES.length + 1 + ); + }); + + test('accumulates count and total RTT per bin', () => { + const profile = sizeProfile([ + { bytes: 160, rttMs: 10 }, + { bytes: 200, rttMs: 20 }, + { bytes: 12060, rttMs: 50 }, + ]); + expect(profile.counts[0]).toBe(2); + expect(profile.totalMs[0]).toBe(30); + expect(profile.counts[RTT_SIZE_BIN_EDGES_BYTES.length]).toBe(1); + expect(profile.totalMs[RTT_SIZE_BIN_EDGES_BYTES.length]).toBe(50); + expect(profile.counts.reduce((a, b) => a + b, 0)).toBe(3); + }); +}); + +describe('histogramRttSamples', () => { + test('bins are [prev edge, edge), first bin is <1ms, last is 5000+', () => { + expect(histogramRttSamples([0, 0.5])[0]).toBe(2); + // A sample exactly on an edge lands in the bin the edge opens. + const atEdge = histogramRttSamples([1]); + expect(atEdge[0]).toBe(0); + expect(atEdge[1]).toBe(1); + const overflow = histogramRttSamples([5000, 60000]); + expect(overflow[RTT_HIST_EDGES_MS.length]).toBe(2); + }); + + test('counts sum to the sample count', () => { + const samples = [0, 1, 3, 7, 59, 128, 438, 1229, 9999]; + const hist = histogramRttSamples(samples); + expect(hist).toHaveLength(RTT_HIST_EDGES_MS.length + 1); + expect(hist.reduce((a, b) => a + b, 0)).toBe(samples.length); + }); +}); + +describe('summarizeRttSamples', () => { + test('returns undefined for an empty bucket', () => { + expect(summarizeRttSamples([])).toBeUndefined(); + }); + + test('single sample collapses every stat to that value', () => { + expect(summarizeRttSamples([7])).toEqual({ + count: 1, + best: 7, + avg: 7, + hist: histWith(7), + p50: 7, + p75: 7, + p90: 7, + p99: 7, + }); + }); + + test('percentiles use the runner convention (nearest-rank via ceil)', () => { + // 1..100 shuffled: pQ must be exactly Q under nearest-rank. + const samples = Array.from({ length: 100 }, (_, i) => i + 1).sort( + () => 0.5 - Math.random() + ); + expect(summarizeRttSamples(samples)).toEqual({ + count: 100, + best: 1, + avg: 50.5, + hist: histogramRttSamples(samples), + p50: 50, + p75: 75, + p90: 90, + p99: 99, + }); + }); + + test('rounds to 0.1ms', () => { + const summary = summarizeRttSamples([1, 2, 2.44]); + expect(summary?.avg).toBe(1.8); + expect(summary?.p99).toBe(2.4); + }); +}); + +describe('summarizeDelayTail', () => { + test('returns undefined for no samples', () => { + expect(summarizeDelayTail([])).toBeUndefined(); + }); + + test('max catches a single stall that pooled percentiles would hide', () => { + // 299 jitter-floor samples plus ONE 800ms stall. + const samples = [...Array.from({ length: 299 }, () => 2), 800]; + const tail = summarizeDelayTail(samples); + expect(tail?.maxMs).toBe(800); + // Even p99 over the pooled run misses a 1-in-300 stall (nearest-rank + // p99 of 300 samples is the 297th) — which is why the runner reports + // per-run max, not pooled percentiles. + expect(tail?.p99Ms).toBe(2); + expect(tail?.count).toBe(300); + expect(tail?.avgMs).toBe(4.7); + }); +}); + +describe('steadyRate', () => { + test('returns undefined when the window cannot define a rate', () => { + expect(steadyRate([])).toBeUndefined(); + expect(steadyRate([{ atMs: 0, bytes: 100 }])).toBeUndefined(); + // Same-instant points: zero span. + expect( + steadyRate([ + { atMs: 5, bytes: 1 }, + { atMs: 5, bytes: 1 }, + ]) + ).toBeUndefined(); + }); + + test('computes chunks/s and KiB/s over the trimmed steady window', () => { + // 100 chunks of 1024B at exactly 10ms spacing → 100 c/s, 100 KiB/s. + const points = Array.from({ length: 100 }, (_, i) => ({ + atMs: i * 10, + bytes: 1024, + })); + const rate = steadyRate(points); + expect(rate?.windowChunks).toBe(80); // 10% trimmed each side + expect(rate?.chunksPerSec).toBe(100); + // Bytes are counted over the window's 79 intervals (the first point's + // bytes predate the window's clock): 79 KiB over 790ms = exactly the + // stream's true steady rate, with no 1/(n-1) inflation. + expect(rate?.kibPerSec).toBe(100); + }); + + test('trimming excludes warmup and drain from the sustained rate', () => { + // A slow first and last chunk (cold start / final flush) that would + // wreck the naive whole-run rate. + const points = [ + { atMs: 0, bytes: 100 }, + ...Array.from({ length: 20 }, (_, i) => ({ + atMs: 1000 + i * 10, + bytes: 100, + })), + { atMs: 10_000, bytes: 100 }, + ]; + const rate = steadyRate(points); + // Steady window covers only the 10ms-spaced middle → ~100 c/s, not the + // ~2 c/s the whole-run span would suggest. + expect(rate?.chunksPerSec).toBeGreaterThan(90); + }); +}); + +describe('computeCdv', () => { + // Chunks written every 10ms, delivered in clumps of three: the first of + // each clump waits for the flush, the other two arrive ~together. + const clumped = (): CdvArrival[] => [ + { seq: 0, writtenAt: 1000, readAt: 1030 }, + { seq: 1, writtenAt: 1010, readAt: 1030 }, + { seq: 2, writtenAt: 1020, readAt: 1031 }, + { seq: 3, writtenAt: 1030, readAt: 1060 }, + { seq: 4, writtenAt: 1040, readAt: 1060 }, + { seq: 5, writtenAt: 1050, readAt: 1061 }, + ]; + + test('clumped delivery reads as negative catch-up plus positive stalls', () => { + const { cdvMs, skippedPairs } = computeCdv(clumped()); + // (readGap - writeGap) per pair: (0-10), (1-10), (29-10), (0-10), (1-10). + expect(cdvMs).toEqual([-10, -9, 19, -10, -9]); + expect(skippedPairs).toBe(0); + }); + + test('equals the telescoping identity cdv_i = CTT_i - CTT_{i-1}', () => { + const arrivals = clumped(); + const ctt = arrivals.map((a) => a.readAt - a.writtenAt); + const { cdvMs } = computeCdv(arrivals); + expect(cdvMs).toEqual(ctt.slice(1).map((v, i) => v - ctt[i])); + // ...and the signed sum telescopes to CTT_last - CTT_first. + expect(cdvMs.reduce((a, b) => a + b, 0)).toBe(ctt[ctt.length - 1] - ctt[0]); + }); + + test('is immune to a constant clock offset between writer and reader', () => { + const skewed = clumped().map((a) => ({ ...a, readAt: a.readAt - 5000 })); + // Reader clock 5s behind the writer: every CTT is negative, CDV is + // untouched — each gap subtracts same-clock stamps. + expect(computeCdv(skewed).cdvMs).toEqual(computeCdv(clumped()).cdvMs); + }); + + test('positive cdv is indexed by the later seq, padded to the stream', () => { + const { positiveBySeq } = computeCdv(clumped()); + expect(positiveBySeq.length).toBe(6); + expect(positiveBySeq[3]).toBe(19); + expect(positiveBySeq.filter((v) => v !== undefined)).toEqual([19]); + }); + + test('counts duplicates, reorders, and non-adjacent pairs', () => { + const arrivals: CdvArrival[] = [ + { seq: 0, writtenAt: 1000, readAt: 1030 }, + { seq: 2, writtenAt: 1020, readAt: 1050 }, // hole: skipped pair + { seq: 1, writtenAt: 1010, readAt: 1051 }, // reorder: skipped pair + { seq: 1, writtenAt: 1010, readAt: 1052 }, // duplicate + not adjacent + ]; + const cdv = computeCdv(arrivals); + expect(cdv.cdvMs).toEqual([]); + expect(cdv.duplicateSeqs).toBe(1); + expect(cdv.reorderedArrivals).toBe(1); + expect(cdv.skippedPairs).toBe(3); + }); + + test('a single chunk has no pair', () => { + const cdv = computeCdv([{ seq: 0, writtenAt: 1000, readAt: 1030 }]); + expect(cdv.cdvMs).toEqual([]); + expect(cdv.skippedPairs).toBe(0); + }); +}); + +describe('mergeRttSummaries', () => { + const summary = (overrides: Partial): BenchRttSummary => ({ + count: 10, + best: 1, + avg: 5, + hist: histWith(5, 10), + p50: 5, + p75: 6, + p90: 8, + p99: 9, + ...overrides, + }); + + test('returns undefined when no iteration produced the bucket', () => { + expect(mergeRttSummaries([])).toBeUndefined(); + expect(mergeRttSummaries([undefined, undefined])).toBeUndefined(); + }); + + test('single summary passes through unchanged', () => { + const s = summary({}); + expect(mergeRttSummaries([undefined, s])).toEqual(s); + }); + + test('count sums, best is the min, avg is count-weighted', () => { + const merged = mergeRttSummaries([ + summary({ count: 10, best: 2, avg: 10 }), + summary({ count: 30, best: 1, avg: 2 }), + ]); + expect(merged?.count).toBe(40); + expect(merged?.best).toBe(1); + expect(merged?.avg).toBe(4); // (10*10 + 2*30) / 40 + }); + + test('histograms merge by elementwise summation (exact)', () => { + const merged = mergeRttSummaries([ + summary({ count: 10, hist: histWith(5, 10) }), + summary({ count: 30, hist: histWith(128, 30) }), + ]); + const expected = histWith(5, 10); + const bin128 = histWith(128, 30); + for (let i = 0; i < expected.length; i++) expected[i] += bin128[i]; + expect(merged?.hist).toEqual(expected); + expect(merged?.hist.reduce((a, b) => a + b, 0)).toBe(40); + }); + + test('percentiles merge as percentile-of-percentiles', () => { + const summaries = Array.from({ length: 10 }, (_, i) => + summary({ p50: i + 1, p90: (i + 1) * 10, p99: (i + 1) * 100 }) + ); + const merged = mergeRttSummaries(summaries); + // p50 over the ten per-iteration p50s (1..10) = 5. + expect(merged?.p50).toBe(5); + // p90 over 10..100 = 90. + expect(merged?.p90).toBe(90); + // p99 over 100..1000 = max of maxes (exact at the tail). + expect(merged?.p99).toBe(1000); + }); +}); diff --git a/workbench/example/workflows/97_bench.ts b/workbench/example/workflows/97_bench.ts index a940ee33a8..3bb1e5edaf 100644 --- a/workbench/example/workflows/97_bench.ts +++ b/workbench/example/workflows/97_bench.ts @@ -27,9 +27,40 @@ // overhead/backpressure. Two payload shapes are supported so the runner can // isolate serialization cost: `'text'` (raw string fragments) and // `'structured'` (AI-SDK-style `{ type: 'text-delta', id, text }` objects). +// - `benchCrttWorkflow` measures per-chunk round-trip time (CRTT), reusing the +// SO setup (paced writer + parallel reader, same deployment, so no clock +// skew beyond intra-Vercel NTP bounds) but embedding `{ seq, writtenAt }` in +// every chunk — the SL scenario's payload-embedded-timestamp trick applied +// to the whole stream. The "round trip" is deployment -> stream backend -> +// reader on the same deployment (one clock domain), not an echo back to the +// writer. The reader stamps each chunk's arrival, computes +// `rtt = Date.now() - chunk.writtenAt`, and aggregates on the deployment +// (see 97_bench_rtt.ts): chunk-index buckets, mean-RTT profiles over stream +// progress and over serialized chunk size, and fixed log-bin histograms — +// compact aggregates instead of hundreds of raw samples. +// - `benchReplayWorkflow` reuses the whole CRTT measurement rig but replays a +// REAL captured stream cadence (97_bench_cadence.ts): every write +// instant and chunk size comes from the capture, so the replay scenario's +// only chosen parameter is the speed multiplier. import { createHook, getWorkflowMetadata, getWritable } from 'workflow'; import { getRun } from 'workflow/api'; +import { BENCH_CADENCES } from './97_bench_cadence'; +import { + type BenchDelayTail, + type BenchRttMeanProfile, + type BenchRttSummary, + type BenchSteadyRate, + type CdvArrival, + computeCdv, + progressProfile, + type RttIndexBucket, + rttIndexBucket, + sizeProfile, + steadyRate, + summarizeDelayTail, + summarizeRttSamples, +} from './97_bench_rtt'; export interface BenchStepTiming { /** Date.now() at step body entry */ @@ -109,6 +140,9 @@ const SL_READY_NAMESPACE = 'bench-sl-ready'; // reader-ready barrier pattern SL uses. const SO_STREAM_NAMESPACE = 'bench-so'; const SO_READY_NAMESPACE = 'bench-so-ready'; +// Dedicated streams for the CRTT scenario, same isolation + barrier pattern. +const CRTT_STREAM_NAMESPACE = 'bench-crtt'; +const CRTT_READY_NAMESPACE = 'bench-crtt-ready'; // Deterministic, variable-length text fragments cycled to approximate real // token-stream traffic (≈4.5 UTF-8 bytes on average, including punctuation and // newline "tokens") while keeping every run byte-for-byte reproducible. @@ -141,6 +175,68 @@ function soChunk( : text; } +/** A self-timestamping CRTT chunk. `text` keeps the payload LLM-shaped (the + * same cycled fragments the SO scenarios stream); the `'sweep'` variant adds + * `pad` so the serialized chunk size rotates across the size buckets. */ +export interface BenchChunkRttDelta { + seq: number; + /** Date.now() in the writer step immediately before this chunk's write */ + writtenAt: number; + text: string; + pad?: string; +} + +/** CRTT payload variant. `'llm'` streams LLM-shaped deltas (a few tens of + * bytes each, so the index numbers stay pure of padding); `'sweep'` pads + * deltas in rotation across log-spaced sizes so mean RTT can be profiled as + * a function of serialized chunk size. (The replay scenario replays real + * captured cadences instead — see {@link benchReplayWorkflow}.) */ +export type BenchChunkRttVariant = 'llm' | 'sweep'; + +/** Reader-side aggregation of one CRTT run: per-bucket summaries computed on + * the deployment (see 97_bench_rtt.ts). Buckets that received no samples are + * absent. */ +/** Chunk delay variation for one run, aggregated in the reader step (see + * computeCdv in 97_bench_rtt.ts for the definition and pairing rules). */ +export interface BenchChunkCdv { + /** Number of seq-adjacent pairs measured. */ + pairs: number; + /** Adjacent arrivals whose seqs weren't consecutive (0 by contract). */ + skippedPairs: number; + /** Tail of POSITIVE cdv — delivery clumps/stalls. Negatives (catch-up) + * balance them by the telescoping identity and are not summarized. */ + positive?: BenchDelayTail; + /** Mean positive cdv per tenth of the stream — localizes where delivery + * clumping/stalls concentrate. */ + progress: BenchRttMeanProfile; +} + +export interface BenchChunkRttResult { + /** Number of chunks the reader received (validated against the request) */ + received: number; + /** All chunks pooled — the headline "average per-chunk RTT" summary. */ + all?: BenchRttSummary; + byIndex: Partial>; + /** Mean RTT per tenth of the stream — the drift/trend readout that fixed + * index buckets cannot provide (see progressProfile in 97_bench_rtt.ts). */ + progress: BenchRttMeanProfile; + /** Mean RTT per log size bin — the size→latency curve (only informative + * for the `'sweep'` variant, whose pad rotation occupies every bin). */ + size: BenchRttMeanProfile; + /** Chunk delay variation (delivery jitter), from RAW timestamps. */ + cdv: BenchChunkCdv; + /** Delivered (reader-side) sustained throughput over the steady window. */ + delivered?: BenchSteadyRate; +} + +// Pad lengths cycled by the CRTT `'sweep'` variant: a log ladder chosen so +// the ~60B base chunk serializes to one representative size per size-profile +// bin (~160B, ~400B, ~760B, ~1.5KB, ~3KB, ~6KB, ~12KB — see +// RTT_SIZE_BIN_EDGES_BYTES in 97_bench_rtt.ts). Rotation decouples size from +// seq: every size appears throughout the stream, so the size profile is not +// confounded with warmup or drift. +const CRTT_SWEEP_PAD_LENGTHS = [100, 340, 700, 1400, 3000, 6000, 12000]; + async function timedNoopStep(index: number): Promise { 'use step'; const kind = stepKind(); @@ -438,3 +534,324 @@ export async function benchSoWorkflow( }, }; } + +/** Reader half of the CRTT scenario. Same attach/ready handshake as + * {@link soReaderStep}, but each received chunk is scored individually: + * `rtt = Date.now() - chunk.writtenAt` (clamped at 0 to absorb tiny + * intra-Vercel clock skew between the writer's and reader's instances) and + * an approximate serialized size (`JSON.stringify` length — the payloads are + * ASCII, so chars ≈ UTF-8 bytes). Raw (unclamped) timestamps are also kept + * in arrival order for chunk delay variation — the skew-free + * delivery-jitter companion metric (see computeCdv). Everything is + * aggregated here in the step, so the workflow returns compact summaries + * rather than one number per chunk. */ +async function crttReaderStep(): Promise { + 'use step'; + const { workflowRunId } = getWorkflowMetadata(); + const reader = getRun(workflowRunId) + .getReadable({ namespace: CRTT_STREAM_NAMESPACE }) + .getReader(); + try { + // Initiate the read BEFORE signalling ready so the stream GET is in flight + // by the time the writer starts (identical to the SL/SO handshake). + const firstRead = reader.read(); + + const ready = getWritable<{ ready: true }>({ + namespace: CRTT_READY_NAMESPACE, + }); + const readyWriter = ready.getWriter(); + await readyWriter.write({ ready: true }); + readyWriter.releaseLock(); + await ready.close(); + + const all: number[] = []; + // RTT per seq (indexed by the chunk's own seq, not arrival order) so the + // progress profile bins by position in the stream even if delivery ever + // reorders. + const rttBySeq: (number | undefined)[] = []; + // RAW timestamps in arrival order for CDV — the clamped RTTs below must + // never feed it (clamping breaks cdv_i = CTT_i - CTT_{i-1} and hides the + // negative catch-up half of every clump). Bytes ride along for the + // delivered-throughput computation. + const arrivals: (CdvArrival & { bytes: number })[] = []; + const sizeSamples: { bytes: number; rttMs: number }[] = []; + const byIndex = new Map(); + let received = 0; + let result = await firstRead; + while (!result.done) { + const receivedAt = Date.now(); + const chunk = result.value; + if ( + !chunk || + typeof chunk.seq !== 'number' || + typeof chunk.writtenAt !== 'number' + ) { + throw new Error( + `bench CRTT reader: malformed chunk ${JSON.stringify(chunk)?.slice(0, 120)}` + ); + } + const rtt = Math.max(0, receivedAt - chunk.writtenAt); + all.push(rtt); + rttBySeq[chunk.seq] = rtt; + // Approximate serialized bytes (ASCII payloads, so chars ≈ bytes). + const bytes = JSON.stringify(chunk).length; + arrivals.push({ + seq: chunk.seq, + writtenAt: chunk.writtenAt, + readAt: receivedAt, + // Extra field beyond CdvArrival — reused for delivered throughput. + bytes, + }); + sizeSamples.push({ bytes, rttMs: rtt }); + const bucket = rttIndexBucket(chunk.seq); + const samples = byIndex.get(bucket); + if (samples) samples.push(rtt); + else byIndex.set(bucket, [rtt]); + received++; + result = await reader.read(); + } + + const summarize = (buckets: Map) => { + const out: Partial> = {}; + for (const [bucket, samples] of buckets) { + out[bucket] = summarizeRttSamples(samples); + } + return out; + }; + const cdv = computeCdv(arrivals); + // Ordered, complete delivery is this bench's contract; a violation is a + // stream-integrity failure, not a latency data point. With no + // duplicates and no reorders, zero skipped pairs plus a first seq of 0 + // makes the received sequence exactly contiguous 0..received-1 — the + // runner's received-count check alone can't distinguish a hole from a + // relabeled range. + if ( + cdv.duplicateSeqs > 0 || + cdv.reorderedArrivals > 0 || + cdv.skippedPairs > 0 || + (arrivals.length > 0 && arrivals[0].seq !== 0) + ) { + throw new Error( + `bench CRTT reader: stream integrity violated (duplicates=${cdv.duplicateSeqs}, reordered=${cdv.reorderedArrivals}, holes=${cdv.skippedPairs}, firstSeq=${arrivals[0]?.seq})` + ); + } + return { + received, + all: summarizeRttSamples(all), + byIndex: summarize(byIndex), + progress: progressProfile(rttBySeq), + size: sizeProfile(sizeSamples), + cdv: { + pairs: cdv.cdvMs.length, + skippedPairs: cdv.skippedPairs, + positive: summarizeDelayTail(cdv.cdvMs.filter((v) => v > 0)), + progress: progressProfile(cdv.positiveBySeq), + }, + delivered: steadyRate( + arrivals.map((a) => ({ atMs: a.readAt, bytes: a.bytes })) + ), + }; + } finally { + reader.cancel().catch(() => {}); + } +} + +/** Writer half of the CRTT scenario: identical pacing to {@link soWriterStep} + * (ready barrier, then `chunkCount` chunks at one per `intervalMs`, writing + * immediately when behind schedule), but every chunk is self-timestamping — + * `writtenAt` is stamped immediately before its write — so the reader can + * compute a per-chunk RTT instead of a whole-stream span. + * + * Also reports write slip: `writtenAt_i - scheduledAt_i`, how late each write + * happened vs its open-loop schedule. This is the producer-stall guard + * per-chunk RTT structurally cannot provide — a write delayed by + * backpressure is stamped late, so its RTT still looks fine (coordinated + * omission), but its slip grows. For slip to mean anything the schedule MUST + * stay absolute from `startedAt` (as below): sleeping a fixed interval after + * each awaited write would re-anchor the schedule to the writes themselves + * (closed-loop) and hide the stall. */ +async function crttWriterStep( + chunkCount: number, + intervalMs: number, + variant: BenchChunkRttVariant +): Promise<{ slip?: BenchDelayTail; achieved?: BenchSteadyRate }> { + 'use step'; + const { workflowRunId } = getWorkflowMetadata(); + const readyReader = getRun<{ ready: true }>(workflowRunId) + .getReadable<{ ready: true }>({ namespace: CRTT_READY_NAMESPACE }) + .getReader(); + try { + await readyReader.read(); + } finally { + readyReader.cancel().catch(() => {}); + } + + const writable = getWritable({ + namespace: CRTT_STREAM_NAMESPACE, + }); + const writer = writable.getWriter(); + const slips: number[] = []; + const writes: { atMs: number; bytes: number }[] = []; + const startedAt = Date.now(); + for (let i = 0; i < chunkCount; i++) { + const scheduledAt = startedAt + (i + 1) * intervalMs; + const delay = scheduledAt - Date.now(); + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + const chunk: BenchChunkRttDelta = { + seq: i, + writtenAt: Date.now(), + text: SO_TEXT_FRAGMENTS[i % SO_TEXT_FRAGMENTS.length], + }; + if (variant === 'sweep') { + chunk.pad = 'x'.repeat( + CRTT_SWEEP_PAD_LENGTHS[i % CRTT_SWEEP_PAD_LENGTHS.length] + ); + } + // Slip is stamped at the same instant as `writtenAt` (just before the + // write is enqueued); the awaited write's own duration surfaces in the + // NEXT chunk's slip when it pushes that chunk past its schedule. + slips.push(Math.max(0, chunk.writtenAt - scheduledAt)); + writes.push({ atMs: chunk.writtenAt, bytes: JSON.stringify(chunk).length }); + await writer.write(chunk); + } + writer.releaseLock(); + await writable.close(); + return { + slip: summarizeDelayTail(slips), + // Achieved (writer-side) sustained rate over the steady window: under + // healthy pacing ≈ the nominal rate; if writes block, this is what the + // producer actually managed. + achieved: steadyRate(writes), + }; +} + +/** + * Scenario 7: per-chunk round-trip time (CRTT), measured entirely on the + * deployment. + * + * Same shape as the SO scenario (paced writer + parallel draining reader on a + * dedicated namespaced stream, reader-ready barrier), but the measurement is + * per chunk rather than per stream: every delta embeds `{ seq, writtenAt }` + * (the SL scenario's payload-embedded-timestamp trick applied to all chunks), + * and the reader computes each chunk's write->read RTT on arrival (the "round + * trip" being deployment -> stream backend -> co-located reader, not an echo + * back to the writer). + * + * Naming: CRTT (chunk ROUND-trip time) is reserved for this same-clock-domain + * setup, where "round" is literally true — the chunk returns to the + * deployment whose clock stamped it. The future production write->read + * metric crosses clocks (producer deployment -> arbitrary consumer) and is a + * one-way trip: that one is CTT (chunk trip time), a separate metric with + * its own clock-skew caveats. Keep the names distinct. The reader aggregates the samples on the deployment + * (see 97_bench_rtt.ts): chunk-index buckets, a per-tenth-of-stream progress + * profile, a per-log-size-bin size profile, and fixed log-bin histograms so + * distributions merge and diff exactly across runs. The `'llm'` variant + * streams the same LLM-shaped deltas as SO (index/progress numbers pure of + * padding); the `'sweep'` variant pads deltas in rotation across log-spaced + * sizes (~160B to ~12KB serialized) so the size profile becomes a + * size->latency curve. + */ +export async function benchCrttWorkflow( + chunkCount: number, + intervalMs: number, + variant: BenchChunkRttVariant = 'llm' +): Promise<{ + crtt: BenchChunkRttResult; + writeSlip?: BenchDelayTail; + achieved?: BenchSteadyRate; +}> { + 'use workflow'; + const [crtt, writer] = await Promise.all([ + crttReaderStep(), + crttWriterStep(chunkCount, intervalMs, variant), + ]); + return { crtt, writeSlip: writer.slip, achieved: writer.achieved }; +} + +/** Writer half of the replay scenario: identical structure to + * {@link crttWriterStep} (ready barrier, absolute open-loop schedule, slip + + * achieved-rate reporting), but the schedule and per-chunk sizes come from a + * REAL captured eve cadence (see 97_bench_cadence.ts) instead of a fixed + * interval: chunk i is scheduled at `startedAt + offsetsMs[i] / speed` and + * padded so its serialized size matches the capture. Missed ticks are never + * re-spread — overdue chunks write back-to-back and the lost time surfaces + * as slip (open-loop; see the crttWriterStep caveat). */ +async function replayWriterStep( + cadenceId: string, + speed: number +): Promise<{ slip?: BenchDelayTail; achieved?: BenchSteadyRate }> { + 'use step'; + const cadence = BENCH_CADENCES[cadenceId]; + if (!cadence) { + throw new Error(`bench replay writer: unknown cadence "${cadenceId}"`); + } + const { workflowRunId } = getWorkflowMetadata(); + const readyReader = getRun<{ ready: true }>(workflowRunId) + .getReadable<{ ready: true }>({ namespace: CRTT_READY_NAMESPACE }) + .getReader(); + try { + await readyReader.read(); + } finally { + readyReader.cancel().catch(() => {}); + } + + const writable = getWritable({ + namespace: CRTT_STREAM_NAMESPACE, + }); + const writer = writable.getWriter(); + const slips: number[] = []; + const writes: { atMs: number; bytes: number }[] = []; + const startedAt = Date.now(); + for (let i = 0; i < cadence.offsetsMs.length; i++) { + const scheduledAt = startedAt + cadence.offsetsMs[i] / speed; + const delay = scheduledAt - Date.now(); + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + const chunk: BenchChunkRttDelta = { + seq: i, + writtenAt: Date.now(), + text: SO_TEXT_FRAGMENTS[i % SO_TEXT_FRAGMENTS.length], + }; + // Pad the delta so its serialized size matches the captured event's + // (~60B envelope of seq/writtenAt/text; exactness beyond a few bytes + // doesn't matter — the doubling size bins absorb it). + const pad = cadence.sizes[i] - 60; + if (pad > 0) chunk.pad = 'x'.repeat(pad); + slips.push(Math.max(0, chunk.writtenAt - scheduledAt)); + writes.push({ atMs: chunk.writtenAt, bytes: JSON.stringify(chunk).length }); + await writer.write(chunk); + } + writer.releaseLock(); + await writable.close(); + return { slip: summarizeDelayTail(slips), achieved: steadyRate(writes) }; +} + +/** + * Scenario 8: cadence replay, measured entirely on the deployment. + * + * Same reader, barrier, and measurement machinery as {@link benchCrttWorkflow} + * (per-chunk CRTT, CDV, profiles, delivered rate), but the writer replays a + * REAL captured eve stream cadence at `speed`x: every write instant and every + * chunk size comes from the capture, so nothing about the workload shape is a + * judgment call except the speed multiplier. Eve's protocol re-ships the + * cumulative message per delta, so sizes ramp through the turn — the + * end-of-turn byte-rate peak is part of the workload, not an accident. + */ +export async function benchReplayWorkflow( + cadenceId: string, + speed: number +): Promise<{ + crtt: BenchChunkRttResult; + writeSlip?: BenchDelayTail; + achieved?: BenchSteadyRate; +}> { + 'use workflow'; + const [crtt, writer] = await Promise.all([ + crttReaderStep(), + replayWriterStep(cadenceId, speed), + ]); + return { crtt, writeSlip: writer.slip, achieved: writer.achieved }; +} diff --git a/workbench/example/workflows/97_bench_cadence.ts b/workbench/example/workflows/97_bench_cadence.ts new file mode 100644 index 0000000000..f6bd4be17c --- /dev/null +++ b/workbench/example/workflows/97_bench_cadence.ts @@ -0,0 +1,75 @@ +// Real captured stream cadences for the replay scenarios. +// +// GENERATED from eve-workflow-stream-replay v1 captures (slimmed to +// per-event {offsetMs, serialized size} + provenance); sources at +// ~/journal/playgrounds/durable-streams/replays/.cadence.json. Do not +// hand-edit; re-capture and regenerate. A re-capture is a NEW workload — +// the id is part of the baseline key, so bump it (new capture file name). +// Re-capture requires the trace-lab plumbing vendored (with restore docs) +// at ~/journal/playgrounds/workflow-server-profiling/eve-plumbing/; +// captures from that setup record dirty:true, meaning exactly that +// plumbing. +// +// Boundary prefixes are load-bearing: eve-* = eve's envelope-protocol +// workload (message-so-far re-ships the cumulative message; sizes RAMP +// 142B → 13KB; ~50x raw delta bytes; the demanding outlier tenant). +// gateway-* = raw provider SSE deltas, no envelope (p50 208B = modal +// production chunk size; the typical customer). -2000t = target +// output-token anchor (~production p50 turn length); metadata carries +// actuals. + +/** One captured cadence: when each chunk was written (offset from the first + * write) and how many bytes it serialized to. */ +export interface BenchCadence { + id: string; + /** Which boundary the capture was taken at: eve = envelope-protocol + * workload, gateway = raw provider SSE chunks. */ + boundary: 'eve' | 'gateway'; + model: string; + capturedAt: string; + /** Version of the eve capture tooling (for gateway captures this is the + * probe's eve version, not a workload property). */ + eveVersion: string; + eveCommit: string; + events: number; + spanMs: number; + totalBytes: number; + offsetsMs: number[]; + sizes: number[]; +} + +export const BENCH_CADENCES: Record = { + // eveCommit '(dirty)': verified fixture-wiring-only dirt (zero + // modifications under packages/eve/src); cadence validity vs 0.30.0 also + // checked by emission diff audit + an empirical nano re-capture. + 'eve-gpt-5.6-sol-2000t': { + id: 'eve-gpt-5.6-sol-2000t', + boundary: 'eve', + model: 'openai/gpt-5.6-sol', + capturedAt: '2026-08-12T20:00:42.830Z', + eveVersion: '0.33.3', + eveCommit: 'ebebdc059345 (dirty)', + events: 2593, + spanMs: 52377, + totalBytes: 17144887, + // biome-ignore format: generated capture data + offsetsMs: [0,0,1,1,4501,4501,4502,4537,4547,4548,4557,4557,4557,4596,4605,4736,4746,4746,4746,4746,4746,4793,4840,4856,4865,4865,4875,4875,4898,4916,4961,4971,5031,5042,5051,5062,5062,5062,5062,5088,5098,5152,5165,5174,5218,5232,5242,5242,5242,5242,5279,5288,5298,5298,5344,5354,5364,5364,5374,5374,5384,5384,5403,5423,5430,5430,5470,5481,5492,5492,5534,5544,5554,5564,5565,5598,5615,5625,5635,5635,5635,5635,5681,5683,5721,5735,5746,5746,5785,5795,5848,5859,5868,5879,5879,5879,5879,5933,5943,5943,5943,5943,5943,5943,5975,5985,5985,5995,6005,6005,6005,6005,6036,6060,6072,6072,6072,6072,6072,6118,6124,6125,6125,6135,6135,6135,6135,6210,6278,6288,6339,6351,6361,6401,6478,6479,6536,6539,6549,6588,6599,6650,6668,6678,6678,6678,6678,6713,6780,6816,6826,6826,6826,6857,6867,6867,6867,6903,6912,6923,6933,6934,6934,6962,6980,6990,6990,7025,7034,7087,7098,7149,7160,7171,7171,7180,7180,7180,7180,7207,7225,7238,7247,7247,7247,7247,7271,7281,7281,7291,7291,7301,7301,7330,7403,7412,7460,7470,7515,7590,7600,7601,7601,7637,7650,7659,7659,7669,7669,7669,7699,7719,7726,7726,7761,7775,7786,7834,7844,7844,7895,7914,7923,7924,7924,7924,7943,8003,8013,8064,8125,8185,8197,8214,8245,8260,8266,8266,8314,8322,8322,8366,8426,8436,8446,8456,8456,8485,8509,8519,8519,8519,8519,8519,8550,8565,8573,8605,8625,8634,8634,8634,8663,8726,8739,8749,8782,8807,8817,8817,8842,8919,8966,9021,9047,9057,9057,9057,9057,9082,9141,9203,9214,9261,9322,9373,9383,9393,9393,9393,9414,9464,9465,9500,9557,9569,9616,9630,9640,9677,9741,9802,9811,9859,9878,9884,9920,9939,9949,9949,9949,9949,9985,9995,9995,10039,10054,10065,10065,10101,10111,10193,10217,10279,10295,10302,10302,10302,10338,10403,10413,10459,10486,10520,10533,10544,10578,10597,10606,10607,10607,10607,10643,10701,10726,10738,10748,10748,10748,10748,10763,10774,10784,10830,10840,10840,10841,10841,10841,10841,10851,10860,10861,10861,10861,10885,10895,10895,10938,10956,10967,10967,11006,11017,11017,11059,11072,11083,11123,11133,11145,11155,11155,11186,11197,11207,11217,11228,11228,11228,11228,11248,11320,11330,11376,11387,11387,11387,11435,11447,11457,11467,11467,11467,11501,11516,11526,11536,11536,11536,11536,11567,11577,11587,11587,11629,11692,11703,11759,11785,11795,11795,11825,11834,11835,11884,11888,11937,12012,12033,12043,12043,12060,12093,12103,12103,12103,12117,12192,12203,12212,12213,12213,12213,12213,12239,12250,12304,12369,12482,12482,12483,12491,12504,12512,12512,12522,12522,12532,12532,12553,12568,12580,12593,12600,12600,12613,12672,12725,12741,12751,12789,12803,12814,12823,12823,12823,12823,12846,12858,12905,12973,12983,13025,13037,13048,13078,13078,13078,13078,13078,13090,13099,13156,13166,13167,13211,13215,13270,13287,13297,13297,13297,13298,13298,13326,13343,13351,13351,13361,13361,13361,13390,13467,13469,13469,13477,13477,13503,13519,13529,13529,13539,13540,13540,13540,13564,13574,13622,13632,13682,13747,13769,13769,13819,13822,13841,13851,13851,13862,13887,13916,13975,13994,14004,14042,14054,14054,14054,14123,14124,14124,14126,14126,14126,14126,14162,14174,14174,14185,14185,14185,14185,14218,14228,14274,14288,14303,14314,14314,14314,14314,14334,14387,14408,14419,14419,14454,14533,14534,14534,14534,14541,14542,14542,14542,14571,14587,14587,14642,14702,14702,14702,14703,14714,14714,14744,14760,14770,14780,14780,14780,14780,14801,14915,14917,14917,14934,14945,14949,14949,14949,14949,14987,14999,15009,15020,15020,15020,15021,15038,15096,15161,15215,15224,15238,15272,15342,15351,15367,15376,15376,15376,15376,15391,15404,15414,15508,15528,15532,15540,15551,15551,15552,15552,15593,15608,15615,15615,15615,15615,15615,15637,15664,15674,15675,15684,15694,15694,15694,15710,15720,15730,15730,15730,15731,15752,15763,15775,15785,15785,15785,15786,15806,15873,15877,15892,15902,15902,15902,15902,15926,15940,15950,15960,15960,15960,15960,15982,15993,16003,16049,16066,16076,16076,16076,16106,16120,16131,16140,16141,16141,16141,16162,16175,16221,16234,16244,16244,16254,16254,16254,16254,16275,16299,16340,16358,16367,16368,16403,16430,16455,16501,16512,16512,16512,16541,16551,16551,16551,16551,16551,16551,16568,16628,16648,16659,16660,16682,16691,16743,16753,16763,16763,16800,16820,16829,16830,16830,16868,16897,16907,16907,16907,17380,17390,17390,17391,17391,17391,17391,17391,17391,17391,17391,17391,17392,17392,17392,17411,17421,17421,17421,17515,17526,17526,17526,17535,17564,17578,17626,17639,17649,17683,17695,17706,17716,17716,17730,17730,17730,17740,17751,17798,17865,17914,17926,17972,18001,18012,18033,18047,18056,18056,18089,18149,18254,18320,18335,18345,18345,18378,18389,18402,18412,18412,18413,18413,18480,18481,18481,18481,18495,19433,19452,19463,19473,19473,19473,19473,19488,19555,19563,19573,19585,19597,19597,19610,19628,19643,19652,19652,19652,19672,19698,19708,19708,19709,19709,19745,19793,19812,19822,19823,19823,19861,19871,19886,19892,19902,19902,19921,19931,19941,19941,19986,20044,20053,20101,20118,20128,20128,20128,20162,20175,20185,20195,20195,20234,20249,20249,20249,20254,20255,20255,20255,20286,20314,20316,20316,20316,20316,20351,20365,20365,20371,20411,20433,20443,20443,20443,20443,20453,20453,20469,20497,20507,20507,20507,20508,20508,20532,20541,20625,20652,20664,20672,20683,20683,20683,20693,20693,20715,20767,20787,20797,20797,20825,20839,20885,20902,20913,20956,20966,20966,20966,20966,21002,21017,21032,21032,21037,21037,21037,21037,21061,21075,21084,21126,21134,21183,21195,21210,21220,21220,21220,21220,21243,21252,21252,21263,21263,21263,21305,21319,21329,21329,21363,21383,21393,21393,21394,21394,21419,21430,21478,21490,21538,21550,21565,21575,21575,21575,21575,21606,21628,21657,21671,21681,21717,21793,21801,21801,21801,21811,21822,21832,21832,21851,21861,21862,21873,21873,21874,21874,21938,21955,22098,22109,22109,22109,22110,22110,22110,22110,22110,22110,22110,22110,22111,22210,22221,22221,22221,22221,22221,22221,22394,22404,22404,22404,22404,22405,22405,22405,22405,22414,22497,22497,22498,22498,22498,22498,22512,22572,22583,22630,22641,22698,22753,22806,22823,22831,22831,22831,22831,22864,22875,22890,22906,22906,22906,22906,22923,22937,22981,23002,23011,23011,23011,23046,23062,23114,23172,23232,23242,23302,23323,23334,23334,23334,23334,23355,23415,23494,23508,23534,23544,23600,23612,23648,23713,23768,23822,23880,23890,23906,23914,23914,23914,23914,23940,23951,23961,23996,24058,24077,24087,24087,24113,24123,24171,24228,24284,24297,24341,24404,24414,24414,24455,24516,24524,24535,24569,24582,24592,24626,24638,24683,24699,24709,24709,24736,24795,24805,24859,24869,24908,24968,24978,24988,24998,24998,24998,24998,25019,25040,25050,25050,25092,25102,25139,25177,25188,25202,25245,25314,25328,25367,25380,25390,25390,25390,25390,25400,25417,25536,25537,25537,25537,25537,25537,25537,25537,25550,25552,25561,25600,25610,25660,25670,25680,25680,25713,25724,25734,25744,25744,25744,25744,25770,25788,25797,25797,25835,25847,25856,25857,25867,25867,25867,25867,25887,25905,25915,25950,25998,26008,26008,26008,26008,26017,26017,26018,26064,26119,26177,26235,26245,26255,26298,26308,26308,26351,26409,26429,26437,26447,26447,26447,26447,26465,26476,26528,26545,26582,26592,26641,26665,26674,26684,26685,26685,26685,26711,26725,26735,26746,26746,26746,26746,27421,27439,27451,27451,27451,27451,27484,27496,27561,27634,27644,27644,27644,27644,27688,27698,27698,27698,27698,27728,27796,27806,27806,27807,27807,27807,27807,27807,27929,27944,27960,27962,27963,27963,27969,27979,28225,28237,28237,28237,28237,28238,28238,28238,28238,28238,28238,28238,28238,28238,28238,28238,28238,28327,28340,28344,28383,28393,28393,28393,28419,28463,28484,28520,28533,28542,28542,28578,28596,28606,28649,28702,28718,28728,28738,28738,28738,28738,28759,28827,28844,28855,28855,28882,28901,28911,28911,28954,28964,28974,28974,28974,28974,29007,29074,29083,29129,29280,29290,29290,29290,29299,29300,29315,29315,29315,29315,29323,29332,29332,29366,29379,29402,29412,29412,29412,29412,29557,29558,29559,29560,29560,29560,29564,29576,29577,29577,29634,29636,29732,29743,29753,29753,29753,29793,29803,29855,29872,29881,29881,29881,29881,29915,29937,29973,29984,29995,30005,30006,30006,30006,30033,30049,30059,30059,30059,30095,30160,30170,30180,30233,30234,30272,30273,30277,30285,30350,30352,30392,30393,30409,30411,30459,30459,30460,30460,30460,30517,30529,30539,30539,30578,30589,30599,30641,30651,30671,30701,30714,30724,30724,30738,30751,30751,30762,30770,30817,30845,30855,30863,30863,30863,30863,30881,30891,30903,30903,30903,30940,30954,30977,30985,30985,30985,30997,31016,31025,31025,31059,31070,31114,31125,31134,31134,31134,31144,31144,31145,31145,31180,31191,31215,31226,31226,31226,31226,31239,31297,31363,31368,31422,31432,31443,31480,31493,31501,31501,31540,31547,31557,31596,31615,31623,31624,31698,31716,31725,31726,31726,31768,31798,31809,31810,31810,31833,31883,31896,31904,31913,31923,31923,31924,31924,31943,31956,31972,31979,31979,32000,32013,32023,32059,32075,32086,32119,32135,32147,32158,32158,32158,32183,32202,32238,32257,32266,32306,32324,32325,32331,32336,32336,32336,32337,32356,32423,32433,32509,32509,32510,32510,32510,32510,32510,32536,32551,32573,32577,32577,32577,32577,32602,32612,32677,32687,32699,32699,32699,32699,32716,32780,32839,32897,32923,32934,32934,32934,32934,32934,32964,32975,32975,32975,33032,33041,33041,33070,33092,33102,33102,33134,33144,33191,33213,33223,33223,33233,33233,33233,33233,33247,33259,33272,33282,33282,33306,33319,33329,33329,33340,33340,33363,33380,33390,33400,33400,33400,33400,33440,33450,33478,33505,33516,33516,33516,33516,33516,33544,33554,33555,34317,34375,34389,34399,34438,34503,34511,34521,34534,34544,34545,34545,34557,34616,34693,34696,34712,34722,34723,34723,34750,34761,34761,34761,34771,34771,34771,34771,34796,34808,34821,34831,34831,34859,34871,34884,34897,34897,34897,34917,34978,34997,35007,35017,35017,35017,35038,35049,35060,35099,35109,35119,35119,35119,35162,35172,35172,35183,35193,35193,35193,35193,35217,35230,35241,35250,35260,35260,35261,35277,35301,35311,35311,35311,35311,35339,35351,35361,35361,35399,35509,35509,35509,35533,35543,35553,35553,35553,35553,35553,35591,35600,35601,35611,35612,35612,35612,35642,35652,35662,35672,35673,35673,35673,35703,35713,35713,36717,36863,36881,36893,36894,36894,36894,36894,36946,36967,36977,36991,37020,37037,37047,37047,37057,37069,37079,37079,37089,37089,37127,37200,37234,37259,37275,37286,37295,37305,37316,37325,37357,37369,37369,37369,37382,37404,37422,37422,37422,37422,37422,37426,37426,37427,37427,37427,37427,37427,37427,37427,37436,37436,37445,37457,37457,37465,37465,37465,37466,37475,37475,37475,37475,37475,37475,37475,37498,37508,37508,37508,37582,37592,37593,37593,37594,37594,37594,37624,37634,37706,37709,37719,37719,37720,37720,37774,37784,37784,37785,37785,37795,37795,37811,37821,37821,37831,37841,37842,37842,37842,37880,37890,37900,37910,37910,37911,37911,37947,37959,37969,37979,37989,38013,38039,38053,38063,38073,38074,38074,38074,38084,38121,38131,38331,38333,38342,38356,38362,38363,38363,38375,38386,38386,38386,38411,38423,38423,38423,38423,38423,38423,38424,38444,38464,38475,38475,38475,38475,38475,38497,38507,38507,38507,38552,38614,38624,38673,38688,38698,38698,38698,38698,38727,38795,38805,38805,38814,38815,38815,38815,38857,38866,38867,38867,38867,38867,38867,38910,38932,38932,38961,38971,39021,39039,39049,39049,39049,39049,39049,39074,39153,39173,39183,39183,39183,39183,39183,39192,39193,39245,39308,39318,39328,39328,39328,39329,39373,39375,39385,39385,39385,39430,39540,39542,39543,39543,39543,39544,39544,39545,39547,39557,39647,39661,39670,39680,39690,39719,39751,39798,39822,39837,39858,39877,39887,39888,39897,39898,39898,39908,39908,39908,39908,39920,39920,39920,39920,39920,39920,39921,39921,39921,39921,39921,39921,39973,40000,40077,40087,40087,40097,40098,40098,40098,40141,40151,40151,40151,40151,40151,40181,40193,40203,40203,40240,40300,40310,40310,40354,40366,40366,40374,40411,40423,40434,40434,40470,40480,40490,40491,40543,40553,40553,40587,40597,40677,40687,40687,40697,40697,40707,40707,40717,40727,40737,40737,40737,40738,40777,40788,40788,40788,40788,40814,41016,41026,41035,41046,41069,41079,41095,41122,41274,41284,41284,41285,41285,41285,41285,41286,41286,41286,41286,41286,41368,41378,41378,41379,41379,41379,41379,41379,41379,41379,41388,41398,41399,41468,41478,41479,41479,41479,41479,41479,41501,41511,41561,41571,41620,41625,41625,41678,41699,41709,41709,41738,41747,41787,41805,41816,41828,41828,41828,41851,41866,41874,41884,41894,41894,41904,41904,41925,41935,41935,41936,41936,41958,42024,42054,42064,42064,42086,42096,42121,42131,42132,42132,42132,42141,42208,42221,42233,42242,42242,42242,42242,42277,42307,42346,42358,42368,42376,42401,42411,42411,42411,42412,42529,42531,42531,42531,42532,42532,42532,42546,42546,42546,42546,42556,42567,42567,42623,42637,42644,42644,42645,42645,42695,42705,42705,42715,42715,42716,42716,42750,42760,42770,42770,42770,42810,42822,42830,42830,42830,42830,42830,42883,42892,42892,42892,42926,42936,42936,42936,42936,42982,43044,43054,43109,43117,43174,43185,43197,43207,43207,43207,43234,43244,43244,43244,43290,43349,43413,43424,43469,43535,43545,43661,43670,43683,43693,43693,43750,43766,43776,43841,43860,43871,43889,43903,43912,43922,43922,43923,43923,44899,44899,44899,44900,44900,44946,44957,44957,44957,44992,45049,45059,45060,45109,45119,45119,45169,45182,45193,45193,45193,45228,45239,45286,45363,45373,45373,45374,45441,45448,45448,45449,45449,45449,45449,45476,45539,45550,45598,45607,45674,45745,45756,45756,45756,45756,45788,45855,45924,45933,45934,45934,45985,46040,46122,46132,46132,46132,46177,46187,46187,46197,46197,46197,46198,46232,46240,46240,46291,46302,46391,46401,46497,46497,46497,46519,46529,46546,46556,46556,46556,46556,46584,46594,46594,46594,46595,46630,46641,46651,46661,46661,46661,46661,46695,46705,46750,46806,46816,46826,46826,46836,46836,46836,46868,46878,46928,46938,46938,46938,46999,47003,47004,47004,47013,47014,47014,47048,47058,47058,47058,47098,47160,47216,47278,47349,47360,47360,47398,47408,47489,47501,47501,47524,47534,47583,47593,47648,47659,47714,47722,47734,47734,47734,47734,47763,47773,47783,47783,47783,47854,47864,47864,47864,47864,47897,47907,47947,47956,48004,48013,48013,48063,48073,48073,48130,48140,48140,48150,48150,48150,48182,48244,48254,48307,48356,48367,48412,48482,48492,48572,48583,48595,48659,48669,48669,48726,48782,48819,48822,48822,48839,48856,48867,48867,48896,48906,48961,48971,48971,49020,49030,49074,49084,49133,49143,49203,49253,49264,49264,49327,49338,49338,49338,49338,49377,49387,49387,49387,49431,49495,49497,49508,49508,49568,49578,49629,49639,49639,49639,49668,49728,49791,49808,49816,49816,49816,49849,49904,49962,50028,50088,50098,50098,50110,50110,50110,50110,50138,50215,50225,50225,50225,50225,50260,50327,50337,50337,50337,50380,50436,50500,50557,50617,50627,50682,50692,50692,50741,50749,50749,50811,50863,50983,50993,50993,51060,51069,51070,51099,51109,51119,51119,51119,51162,51172,51214,51236,51247,51257,51258,51258,51258,51272,51290,51300,51301,51301,51352,51362,51362,51415,51424,51425,51440,51450,51450,51450,51451,51483,51532,51542,51542,51542,51568,51632,51643,51661,51697,51706,51718,51727,51727,51736,51760,51770,51770,51770,51807,51886,51946,51957,51984,52062,52072,52330,52331,52377,52377], + // biome-ignore format: generated capture data + sizes: [218,142,739,187,198,205,211,218,228,228,230,249,254,266,264,270,271,274,277,296,289,301,302,312,315,314,328,322,329,341,339,354,355,360,371,373,370,375,396,388,404,399,411,414,435,430,446,446,443,462,470,467,472,476,486,481,484,486,488,494,492,497,502,510,520,516,532,533,534,541,562,557,569,563,574,572,584,589,593,610,613,612,611,616,625,623,629,639,634,641,657,661,662,672,671,678,693,689,696,695,699,708,704,715,719,716,721,722,722,727,730,733,738,736,737,742,743,755,750,753,755,761,761,769,770,789,792,791,800,814,810,819,819,822,833,842,844,845,848,853,851,868,891,893,890,893,902,898,905,919,914,932,934,930,943,946,943,950,966,970,987,981,996,999,1013,1019,1029,1022,1037,1037,1051,1052,1057,1074,1065,1070,1075,1083,1083,1086,1089,1100,1113,1109,1112,1133,1127,1130,1135,1137,1142,1156,1151,1151,1162,1172,1182,1179,1194,1213,1205,1208,1211,1218,1231,1224,1243,1243,1256,1253,1256,1261,1259,1268,1267,1273,1291,1296,1294,1311,1309,1319,1334,1339,1333,1348,1352,1364,1369,1372,1383,1387,1389,1401,1401,1413,1415,1412,1425,1432,1437,1441,1438,1459,1460,1456,1465,1476,1482,1477,1492,1496,1512,1503,1518,1526,1534,1536,1543,1547,1552,1563,1557,1562,1575,1579,1575,1588,1603,1604,1616,1615,1633,1648,1649,1648,1651,1662,1677,1676,1677,1694,1698,1699,1713,1718,1721,1734,1731,1730,1733,1745,1758,1768,1766,1788,1785,1792,1806,1808,1820,1833,1840,1833,1842,1843,1856,1862,1877,1868,1873,1886,1886,1892,1897,1898,1917,1920,1919,1924,1936,1932,1940,1947,1943,1948,1959,1976,1974,1971,1978,1978,1986,1991,1998,2002,2007,2010,2010,2010,2013,2015,2017,2027,2047,2040,2047,2046,2050,2059,2055,2068,2071,2068,2073,2082,2082,2093,2094,2092,2093,2098,2099,2099,2102,2102,2107,2108,2116,2115,2125,2135,2131,2139,2136,2143,2153,2166,2168,2166,2186,2189,2188,2197,2197,2206,2206,2211,2217,2220,2222,2227,2233,2240,2250,2246,2254,2253,2257,2258,2258,2261,2265,2270,2268,2269,2274,2283,2297,2289,2304,2304,2304,2306,2314,2311,2338,2340,2346,2351,2353,2357,2361,2367,2364,2367,2375,2384,2391,2390,2393,2404,2407,2414,2414,2433,2428,2439,2438,2441,2454,2456,2455,2475,2472,2470,2477,2495,2498,2507,2517,2528,2524,2531,2544,2537,2542,2559,2563,2566,2567,2570,2581,2596,2599,2598,2599,2601,2603,2613,2615,2621,2622,2635,2641,2640,2645,2649,2647,2654,2664,2677,2685,2690,2689,2702,2708,2707,2732,2726,2730,2728,2743,2741,2748,2755,2763,2764,2790,2789,2792,2801,2802,2810,2814,2821,2835,2833,2843,2852,2846,2853,2859,2866,2886,2889,2883,2890,2900,2913,2907,2907,2912,2911,2916,2941,2935,2939,2937,2942,2951,2961,2959,2982,2971,2980,2989,2990,3000,2999,3012,3018,3025,3028,3031,3037,3036,3040,3039,3044,3043,3044,3049,3048,3049,3054,3053,3054,3059,3058,3061,3065,3072,3071,3073,3081,3084,3086,3099,3105,3104,3107,3112,3110,3117,3137,3137,3147,3157,3150,3153,3153,3154,3157,3157,3158,3161,3161,3168,3166,3169,3175,3173,3180,3184,3188,3186,3193,3219,3220,3220,3226,3241,3249,3254,3249,3256,3260,3274,3283,3280,3301,3303,3304,3312,3316,3321,3324,3332,3346,3352,3350,3363,3367,3369,3379,3382,3381,3405,3408,3410,3419,3420,3426,3429,3445,3453,3446,3451,3462,3479,3481,3486,3486,3505,3510,3504,3509,3514,3520,3539,3533,3538,3546,3560,3556,3557,3561,3560,3567,3575,3575,3584,3590,3606,3611,3605,3610,3611,3611,3616,3617,3625,3628,3628,3634,3640,3647,3647,3646,3655,3652,3657,3660,3661,3665,3670,3678,3694,3703,3703,3706,3724,3719,3734,3727,3734,3740,3739,3743,3742,3749,3759,3758,3763,3775,3781,3780,3787,3788,3799,3794,3811,3808,3817,3821,3826,3835,3830,3839,3838,3852,3854,3855,3860,3864,3862,3871,3882,3880,3883,3886,3885,3900,3898,3911,3925,3922,3936,3943,3939,3953,3959,3968,3966,3983,3993,3995,3999,3996,4001,4002,4010,4019,4028,4026,4029,4030,4034,4037,4048,4069,4079,4103,4093,4105,4115,4114,4128,4137,4130,4135,4142,4141,4141,4146,4145,4154,4155,4156,4166,4176,4182,4177,4204,4198,4202,4200,4207,4219,4221,4228,4224,4237,4246,4248,4245,4252,4264,4274,4281,4283,4288,4309,4306,4319,4320,4333,4334,4349,4345,4362,4360,4358,4367,4366,4374,4381,4395,4393,4397,4395,4402,4406,4410,4422,4418,4422,4421,4432,4432,4443,4442,4445,4456,4469,4463,4463,4472,4473,4484,4485,4503,4510,4511,4516,4533,4524,4539,4539,4545,4560,4554,4554,4559,4558,4563,4588,4582,4586,4590,4594,4604,4607,4620,4617,4624,4621,4634,4635,4633,4638,4649,4648,4661,4663,4668,4667,4671,4672,4678,4676,4677,4682,4681,4686,4691,4697,4698,4703,4701,4702,4707,4706,4713,4713,4719,4717,4718,4721,4721,4722,4725,4727,4733,4731,4732,4737,4742,4752,4749,4751,4759,4768,4769,4777,4777,4792,4789,4812,4801,4808,4828,4838,4839,4861,4855,4859,4857,4866,4865,4877,4882,4890,4889,4896,4895,4899,4900,4904,4903,4906,4910,4909,4912,4916,4915,4918,4922,4921,4924,4928,4929,4933,4934,4936,4952,4949,4954,4956,4959,4966,4979,4974,4988,4986,4991,4989,4998,5001,5005,5017,5031,5036,5038,5035,5040,5055,5052,5061,5061,5068,5081,5078,5091,5099,5099,5113,5118,5117,5128,5131,5144,5151,5149,5156,5161,5160,5164,5165,5171,5169,5174,5173,5182,5183,5188,5194,5193,5195,5199,5208,5208,5213,5223,5218,5239,5232,5232,5237,5236,5237,5238,5241,5253,5252,5253,5261,5272,5266,5283,5300,5307,5307,5336,5327,5331,5345,5344,5344,5348,5363,5356,5363,5375,5381,5386,5400,5392,5403,5413,5415,5432,5427,5442,5443,5448,5457,5464,5469,5473,5470,5473,5473,5480,5490,5503,5501,5515,5528,5519,5528,5533,5540,5552,5549,5562,5572,5575,5571,5582,5596,5596,5613,5610,5632,5625,5634,5644,5652,5646,5657,5667,5681,5678,5676,5683,5701,5692,5701,5708,5708,5715,5722,5730,5739,5735,5743,5750,5746,5755,5758,5760,5777,5769,5784,5778,5785,5797,5791,5800,5805,5812,5816,5827,5827,5825,5840,5840,5854,5855,5872,5869,5873,5887,5890,5896,5904,5907,5912,5909,5912,5912,5923,5927,5926,5934,5945,5943,5948,5950,5965,5972,5968,5982,5990,5996,5999,6002,6014,6020,6025,6035,6033,6040,6043,6057,6062,6059,6071,6082,6081,6083,6088,6102,6109,6119,6112,6113,6114,6115,6116,6117,6118,6119,6120,6121,6122,6123,6132,6129,6144,6152,6146,6147,6154,6160,6157,6164,6168,6166,6173,6191,6182,6187,6196,6218,6220,6214,6215,6224,6237,6242,6251,6251,6269,6288,6283,6286,6298,6306,6300,6301,6302,6307,6312,6314,6323,6335,6340,6335,6340,6359,6360,6360,6365,6383,6382,6383,6392,6402,6410,6404,6407,6417,6419,6421,6436,6435,6433,6436,6444,6449,6464,6466,6480,6480,6477,6492,6492,6504,6506,6509,6521,6537,6535,6538,6550,6550,6552,6573,6563,6578,6586,6586,6594,6592,6592,6601,6602,6605,6610,6622,6620,6621,6629,6630,6641,6640,6649,6663,6663,6664,6665,6687,6679,6703,6696,6707,6722,6719,6753,6742,6740,6749,6754,6765,6765,6771,6768,6783,6783,6795,6809,6814,6830,6834,6842,6836,6847,6853,6865,6866,6875,6878,6877,6891,6893,6908,6910,6914,6931,6930,6931,6948,6958,6960,6956,6961,6964,6963,6968,6971,6974,6989,6992,6996,7005,7014,7032,7021,7024,7028,7039,7040,7050,7049,7078,7076,7079,7098,7096,7113,7110,7108,7119,7123,7132,7143,7142,7146,7158,7156,7159,7170,7173,7174,7179,7193,7188,7188,7199,7213,7211,7209,7220,7222,7228,7227,7235,7238,7244,7259,7263,7274,7267,7278,7290,7297,7297,7311,7310,7326,7324,7340,7338,7336,7337,7344,7350,7351,7368,7370,7382,7383,7380,7387,7393,7392,7392,7397,7396,7403,7407,7411,7413,7418,7422,7430,7436,7442,7444,7456,7476,7472,7477,7476,7490,7490,7496,7493,7504,7510,7524,7522,7520,7527,7537,7558,7554,7551,7552,7561,7566,7563,7570,7568,7573,7580,7583,7595,7601,7604,7605,7604,7605,7614,7615,7620,7624,7642,7647,7655,7653,7652,7653,7662,7661,7661,7666,7665,7688,7682,7683,7687,7686,7697,7693,7694,7701,7709,7709,7718,7724,7726,7738,7738,7742,7750,7762,7763,7780,7779,7782,7794,7794,7804,7817,7815,7813,7814,7821,7827,7844,7853,7851,7849,7858,7861,7875,7870,7870,7875,7874,7879,7892,7892,7896,7910,7905,7911,7923,7921,7938,7930,7939,7942,7946,7952,7953,7962,7962,7977,7976,7980,7988,7994,7996,8008,8016,8014,8029,8030,8033,8035,8040,8052,8054,8063,8064,8076,8080,8090,8084,8093,8096,8098,8105,8106,8107,8113,8111,8116,8117,8121,8120,8127,8125,8146,8157,8166,8166,8171,8176,8189,8194,8197,8228,8218,8224,8229,8232,8242,8237,8252,8250,8255,8263,8267,8280,8295,8291,8299,8309,8305,8319,8313,8318,8321,8320,8325,8326,8342,8343,8356,8353,8368,8375,8375,8373,8378,8393,8390,8403,8405,8408,8424,8438,8438,8440,8446,8465,8463,8476,8479,8491,8508,8497,8500,8504,8513,8525,8522,8529,8540,8544,8548,8547,8561,8569,8569,8571,8574,8579,8597,8596,8593,8608,8604,8604,8609,8608,8613,8612,8613,8620,8622,8621,8624,8628,8627,8634,8632,8637,8654,8646,8649,8655,8657,8656,8667,8677,8675,8698,8691,8690,8693,8705,8708,8717,8720,8719,8719,8724,8723,8728,8727,8730,8736,8738,8737,8744,8750,8751,8766,8763,8772,8786,8784,8782,8783,8790,8792,8805,8807,8808,8825,8821,8828,8825,8832,8834,8843,8841,8849,8848,8850,8850,8857,8861,8859,8864,8863,8868,8867,8872,8877,8875,8886,8884,8892,8899,8897,8899,8905,8909,8907,8918,8920,8918,8919,8922,8930,8941,8935,8936,8943,8947,8945,8956,8958,8956,8957,8960,8968,8975,8973,8973,8974,8981,8985,8983,8994,8996,8994,8995,8998,9006,9013,9011,9011,9012,9019,9023,9021,9032,9034,9032,9037,9046,9060,9054,9054,9065,9067,9065,9084,9076,9081,9088,9095,9109,9103,9103,9106,9112,9116,9126,9127,9125,9126,9139,9144,9158,9154,9155,9161,9165,9163,9174,9178,9181,9179,9192,9197,9211,9205,9205,9208,9208,9209,9212,9212,9219,9217,9218,9221,9227,9233,9240,9254,9252,9250,9253,9261,9272,9270,9273,9282,9280,9282,9286,9285,9292,9292,9296,9301,9301,9319,9313,9313,9318,9319,9319,9322,9328,9326,9333,9349,9347,9363,9357,9365,9372,9386,9380,9380,9383,9389,9393,9391,9410,9408,9406,9409,9415,9421,9438,9429,9432,9434,9436,9436,9437,9440,9440,9455,9459,9457,9465,9480,9472,9481,9490,9491,9489,9492,9494,9496,9508,9525,9527,9526,9527,9531,9530,9537,9535,9542,9542,9548,9546,9559,9580,9568,9575,9591,9595,9604,9604,9604,9606,9606,9607,9610,9610,9617,9615,9622,9624,9637,9637,9641,9649,9671,9659,9660,9669,9682,9691,9684,9691,9707,9719,9722,9721,9720,9723,9729,9733,9747,9756,9763,9765,9764,9780,9781,9784,9796,9802,9799,9813,9807,9814,9822,9824,9824,9828,9827,9834,9832,9837,9850,9844,9847,9853,9857,9867,9864,9864,9869,9868,9873,9874,9884,9906,9894,9903,9906,9904,9913,9914,9923,9927,9932,9939,9949,9943,9944,9945,9946,9947,9950,9950,9951,9958,9956,9967,9975,9988,9984,10003,10012,10004,10013,10032,10028,10034,10047,10044,10069,10065,10082,10081,10080,10082,10084,10086,10092,10090,10095,10100,10114,10111,10136,10130,10134,10138,10142,10146,10148,10153,10159,10160,10167,10172,10177,10174,10183,10190,10196,10192,10203,10215,10210,10220,10228,10223,10232,10235,10247,10243,10251,10256,10257,10278,10284,10291,10303,10297,10301,10302,10306,10317,10320,10321,10326,10346,10356,10361,10379,10378,10374,10385,10393,10400,10413,10407,10417,10429,10442,10441,10448,10450,10456,10473,10474,10478,10477,10481,10480,10483,10483,10486,10488,10490,10492,10494,10494,10497,10499,10505,10507,10510,10511,10519,10522,10520,10527,10525,10532,10534,10537,10536,10549,10556,10559,10562,10576,10577,10574,10581,10579,10582,10582,10601,10599,10607,10603,10610,10614,10612,10615,10615,10632,10639,10633,10636,10638,10640,10650,10660,10664,10666,10674,10694,10701,10703,10699,10710,10710,10713,10732,10729,10727,10734,10742,10740,10746,10752,10763,10763,10771,10781,10779,10794,10793,10799,10812,10813,10822,10829,10838,10844,10843,10850,10859,10862,10865,10875,10874,10879,10877,10892,10886,10889,10891,10893,10897,10914,10914,10917,10933,10933,10938,10941,10951,10958,10957,10964,10969,10974,10981,10977,10982,10985,10994,11016,11012,11019,11015,11028,11031,11042,11040,11057,11069,11072,11067,11072,11083,11084,11092,11100,11099,11112,11112,11118,11123,11122,11124,11126,11132,11136,11134,11139,11142,11149,11156,11160,11165,11168,11176,11180,11199,11197,11194,11205,11203,11221,11213,11224,11230,11242,11235,11248,11251,11264,11265,11266,11271,11283,11297,11292,11311,11302,11327,11324,11321,11328,11334,11337,11343,11350,11362,11363,11360,11371,11379,11376,11376,11379,11381,11387,11385,11392,11404,11414,11417,11421,11426,11423,11432,11433,11436,11451,11448,11469,11469,11465,11468,11470,11476,11484,11486,11494,11490,11493,11495,11513,11511,11527,11529,11539,11541,11546,11545,11545,11550,11559,11557,11559,11561,11565,11572,11573,11578,11582,11588,11587,11595,11608,11609,11618,11621,11624,11626,11637,11640,11641,11654,11656,11661,11676,11678,11674,11677,11679,11703,11704,11705,11711,11726,11724,11728,11728,11738,11748,11748,11768,11766,11767,11784,11780,11787,11804,11809,11825,11821,11833,11835,11840,11849,11854,11866,11863,11870,11877,11895,11889,11904,11897,11900,11902,11908,11922,11929,11927,11942,11951,11944,11953,11960,11976,11967,11974,11994,11992,11999,12005,12001,12012,12010,12012,12028,12033,12044,12043,12051,12067,12074,12067,12078,12090,12097,12095,12108,12114,12113,12120,12117,12134,12147,12138,12145,12157,12157,12163,12170,12186,12185,12184,12198,12198,12208,12209,12219,12222,12229,12247,12249,12244,12267,12261,12276,12281,12290,12284,12307,12303,12309,12314,12331,12326,12341,12334,12343,12350,12364,12372,12379,12373,12394,12403,12415,12422,12431,12424,12433,12444,12442,12455,12461,12464,12473,12470,12484,12478,12485,12491,12504,12537,12538,12537,12536,12544,12553,12548,12559,12571,12570,12568,12585,12594,12589,12601,12618,12626,12624,12636,12644,12646,12643,12646,12648,12662,12658,12672,12674,12681,12693,12692,12702,12709,12716,12723,12724,12730,12727,12736,12749,12752,12748,12751,12753,12767,12773,12784,12785,12788,12808,12808,12804,12817,12816,12829,12831,12834,12840,12843,12859,12861,12871,12865,12882,12875,12886,12902,12913,12910,12926,12922,12927,12945,12950,12952,12961,12964,12973,12980,12995,12998,13001,13014,13013,13021,13027,13037,13031,13030,376,144,195], + }, + 'gateway-gpt-5.4-nano-2000t': { + id: 'gateway-gpt-5.4-nano-2000t', + boundary: 'gateway', + model: 'openai/gpt-5.4-nano', + capturedAt: '2026-08-12', + eveVersion: '0.33.3', + eveCommit: '', + events: 1765, + spanMs: 19943, + totalBytes: 330009, + // biome-ignore format: generated capture data + offsetsMs: [0,11,20,31,41,56,66,77,86,96,108,128,163,173,183,194,204,214,224,239,247,263,280,291,304,317,339,340,351,358,368,378,393,401,411,427,448,458,473,497,507,518,529,537,548,557,577,590,598,606,620,626,637,648,668,676,687,697,707,723,734,746,754,764,774,784,797,806,817,826,840,847,860,867,876,886,897,912,924,934,949,954,964,974,984,994,1004,1014,1025,1034,1044,1055,1065,1075,1085,1095,1106,1116,1125,1142,1152,1163,1173,1183,1208,1218,1232,1242,1253,1278,1288,1298,1310,1320,1331,1341,1351,1361,1371,1390,1400,1410,1421,1431,1441,1463,1475,1482,1492,1518,1528,1538,1559,1567,1577,1588,1597,1608,1619,1632,1638,1649,1661,1668,1678,1689,1699,1710,1719,1730,1746,1756,1761,1774,1782,1792,1802,1812,1822,1834,1842,1855,1870,1875,1886,1896,1907,1917,1926,1936,1946,1969,1979,1989,1999,2009,2022,2034,2043,2053,2075,2085,2096,2105,2115,2126,2137,2148,2156,2166,2179,2189,2200,2210,2223,2233,2252,2254,2270,2284,2294,2305,2315,2325,2336,2345,2361,2373,2383,2394,2407,2416,2426,2436,2446,2461,2471,2494,2505,2515,2525,2542,2555,2583,2592,2602,2612,2623,2634,2643,2653,2663,2677,2687,2703,2708,2725,2736,2746,2756,2766,2777,2787,2802,2812,2822,2836,2847,2857,2868,2877,2887,2898,2910,2917,2927,2941,2948,2959,2968,2978,2988,2998,3019,3025,3035,3046,3057,3067,3077,3087,3098,3107,3117,3141,3151,3161,3171,3181,3191,3201,3212,3222,3232,3263,3273,3283,3293,3303,3316,3325,3334,3345,3355,3364,3385,3396,3405,3415,3425,3435,3445,3456,3466,3477,3486,3496,3506,3516,3526,3536,3546,3556,3568,3576,3588,3618,3628,3638,3662,3672,3682,3692,3702,3713,3723,3733,3743,3753,3767,3778,3788,3798,3809,3818,3828,3839,3851,3858,3868,3878,3898,3908,3918,3928,3938,3948,3958,3968,3978,3988,3998,4008,4019,4029,4041,4051,4065,4075,4087,4098,4108,4118,4128,4138,4148,4158,4168,4178,4189,4199,4209,4225,4235,4247,4255,4265,4275,4300,4311,4320,4330,4341,4351,4361,4371,4381,4391,4401,4411,4421,4431,4441,4459,4468,4478,4489,4499,4512,4519,4529,4543,4552,4560,4575,4589,4592,4607,4610,4633,4661,4663,4671,4682,4691,4701,4711,4722,4732,4742,4752,4762,4773,4782,4792,4803,4812,4822,4833,4843,4853,4863,4873,4883,4894,4904,4914,4927,4937,4948,4957,4968,4978,4988,4998,5009,5020,5030,5039,5057,5067,5077,5087,5097,5107,5128,5139,5148,5159,5170,5179,5189,5199,5209,5219,5229,5239,5249,5261,5269,5279,5295,5304,5314,5325,5344,5348,5359,5370,5378,5389,5398,5408,5418,5432,5575,5600,5653,5674,5905,5916,5926,5944,6088,6102,6122,6132,6143,6152,6162,6184,6185,6192,6203,6213,6223,6243,6244,6262,6264,6274,6285,6294,6304,6315,6325,6335,6346,6355,6368,6376,6386,6396,6413,6460,6461,6462,6462,6470,6476,6487,6504,6507,6518,6527,6537,6547,6557,6568,6579,6588,6601,6610,6619,6631,6657,6667,6681,6687,6697,6731,6732,6733,6737,6748,6759,6768,6778,6788,6801,6808,6819,6829,6843,6849,6859,6869,6879,6907,6909,7060,7062,7063,7063,7063,7063,7064,7064,7070,7223,7224,7224,7225,7226,7226,7226,7226,7226,7227,7248,7250,7257,7274,7275,7293,7296,7309,7319,7329,7340,7350,7363,7369,7412,7414,7414,7415,7420,7431,7441,7455,7460,7470,7486,7491,7502,7511,7523,7531,7541,7551,7561,7571,7581,7591,7601,7618,7625,7632,7644,7654,7664,7675,7685,7699,7710,7720,7730,7740,7752,7760,7770,7780,7790,7801,7811,7821,7833,7845,7854,7863,7874,7884,7894,7904,7914,7924,7934,7944,7954,7964,7975,7985,7994,8004,8014,8025,8035,8045,8057,8065,8076,8085,8096,8106,8120,8126,8140,8147,8157,8167,8177,8188,8197,8210,8217,8244,8255,8264,8284,8286,8304,8306,8314,8324,8335,8347,8357,8367,8377,8387,8398,8408,8419,8428,8438,8449,8459,8469,8479,8492,8499,8509,8519,8534,8540,8553,8562,8573,8583,8596,8610,8628,8642,8660,8675,8698,8705,8715,8726,8736,8746,8758,8766,8776,8786,8797,8807,8817,8827,8837,8847,8858,8869,8878,8888,8898,8910,8918,8928,8938,8949,8959,8968,8979,8989,8999,9012,9023,9032,9042,9054,9062,9076,9086,9097,9106,9117,9125,9140,9145,9153,9163,9173,9183,9194,9204,9214,9224,9234,9244,9254,9267,9277,9303,9304,9305,9316,9326,9336,9348,9356,9369,9378,9388,9397,9406,9418,9428,9438,9448,9459,9469,9481,9496,9501,9510,9519,9530,9540,9551,9560,9575,9584,9594,9607,9617,9631,9648,9658,9677,9688,9702,9709,9719,9729,9739,9750,9761,9770,9780,9790,9800,9811,9820,9830,9842,9851,9873,9873,9883,9893,9903,9914,9924,9934,9945,9955,9965,9975,9985,9995,10005,10020,10030,10040,10050,10060,10074,10084,10095,10104,10115,10132,10142,10152,10167,10173,10183,10193,10203,10213,10223,10233,10243,10254,10264,10292,10305,10313,10323,10333,10345,10356,10365,10376,10386,10395,10405,10418,10426,10436,10446,10457,10466,10476,10486,10497,10507,10517,10528,10537,10550,10561,10569,10579,10589,10599,10609,10619,10629,10639,10649,10667,10672,10682,10691,10702,10712,10722,10732,10742,10752,10762,10772,10782,10792,10802,10813,10824,10833,10844,10853,10863,10874,10884,10894,10904,10914,10924,10934,10945,10955,10965,10975,10986,10996,11006,11016,11026,11037,11048,11060,11068,11078,11090,11109,11112,11121,11131,11141,11151,11161,11172,11182,11192,11202,11214,11223,11233,11247,11253,11267,11273,11283,11293,11305,11314,11327,11336,11348,11358,11369,11378,11389,11398,11410,11418,11429,11439,11449,11460,11469,11480,11489,11499,11510,11520,11530,11542,11550,11561,11570,11580,11591,11601,11614,11622,11631,11642,11652,11662,11673,11682,11693,11706,11713,11724,11734,11748,11753,11764,11774,11792,11807,11810,11823,11842,11845,11854,11863,11874,11884,11894,11904,11914,11924,11937,11947,11957,11968,11978,11988,12000,12010,12020,12030,12040,12050,12066,12076,12086,12096,12107,12118,12126,12137,12147,12158,12167,12177,12190,12200,12232,12233,12233,12239,12249,12258,12268,12278,12288,12298,12309,12319,12330,12340,12350,12362,12373,12383,12393,12403,12413,12423,12433,12443,12453,12464,12475,12483,12494,12504,12517,12527,12534,12544,12554,12565,12575,12587,12598,12608,12619,12629,12648,12651,12659,12670,12680,12690,12700,12711,12722,12738,12746,12757,12772,12776,12786,12796,12806,12817,12827,12837,12847,12857,12869,12878,12888,12898,12909,12919,12929,12941,12964,12966,12971,12981,12991,13001,13011,13021,13032,13041,13052,13063,13074,13082,13092,13102,13112,13123,13145,13146,13153,13163,13173,13184,13197,13207,13217,13227,13238,13248,13258,13268,13278,13289,13304,13308,13318,13328,13339,13350,13359,13374,13384,13395,13405,13415,13425,13435,13446,13456,13466,13476,13486,13496,13506,13517,13526,13539,13549,13560,13570,13580,13590,13600,13610,13620,13630,13641,13651,13661,13671,13681,13691,13705,13715,13727,13736,13746,13756,13771,13782,13795,13802,13813,13823,13834,13845,13856,13864,13874,13886,13895,13905,13915,13926,13935,13945,13955,13967,13978,13990,14005,14020,14031,14044,14051,14061,14072,14082,14092,14102,14112,14123,14133,14144,14155,14164,14174,14197,14207,14220,14227,14237,14249,14261,14271,14281,14291,14308,14318,14328,14340,14348,14359,14375,14381,14392,14402,14412,14423,14432,14443,14454,14464,14474,14484,14494,14505,14517,14525,14539,14550,14559,14569,14579,14589,14599,14610,14620,14630,14640,14655,14660,14670,14681,14692,14701,14711,14721,14732,14742,14752,14762,14772,14784,14793,14803,14815,14825,14837,14847,14857,14868,14878,14888,14898,14910,14928,14940,14954,14969,14984,14998,15014,15029,15045,15060,15067,15077,15088,15097,15107,15118,15128,15138,15148,15158,15169,15180,15188,15199,15209,15219,15229,15240,15250,15260,15270,15280,15297,15309,15315,15325,15335,15346,15355,15368,15378,15405,15406,15415,15443,15453,15463,15473,15487,15494,15504,15516,15525,15535,15545,15555,15567,15578,15587,15598,15609,15618,15629,15639,15662,15669,15681,15691,15702,15726,15740,15751,15766,15775,15785,15795,15806,15816,15830,15836,15846,15857,15869,15878,15888,15903,15911,15922,15931,15940,15951,15960,15972,15981,15991,16001,16011,16042,16052,16070,16081,16096,16109,16118,16127,16147,16148,16156,16166,16182,16187,16197,16209,16219,16233,16239,16251,16261,16271,16281,16291,16302,16312,16322,16336,16349,16359,16367,16379,16388,16398,16408,16418,16430,16441,16451,16461,16471,16481,16493,16501,16511,16522,16537,16541,16553,16565,16574,16584,16594,16618,16627,16637,16647,16660,16668,16679,16688,16698,16708,16718,16729,16738,16749,16759,16770,16779,16789,16800,16812,16819,16830,16839,16850,16860,16891,16892,16897,16907,16917,16927,16938,16949,16966,16969,16977,16988,16998,17008,17020,17031,17044,17055,17065,17075,17086,17095,17105,17117,17125,17135,17152,17158,17165,17196,17196,17196,17207,17215,17225,17238,17246,17257,17266,17277,17301,17302,17309,17316,17327,17337,17347,17357,17369,17377,17388,17398,17411,17418,17428,17438,17449,17459,17469,17490,17491,17499,17510,17522,17531,17541,17552,17562,17573,17588,17606,17618,17628,17638,17648,17658,17669,17679,17689,17699,17710,17720,17758,17759,17767,17779,17788,17798,17809,17825,17835,17845,17856,17870,17875,17889,17897,17907,17919,17927,17938,17947,17958,17968,17978,17988,17998,18009,18018,18028,18044,18049,18070,18081,18090,18100,18110,18120,18130,18140,18161,18173,18193,18203,18211,18227,18253,18269,18275,18284,18294,18305,18314,18325,18336,18347,18359,18369,18380,18394,18400,18409,18420,18430,18442,18450,18461,18474,18485,18495,18506,18518,18529,18539,18551,18559,18570,18580,18590,18600,18610,18621,18631,18641,18651,18661,18671,18681,18691,18701,18711,18721,18731,18742,18752,18762,18772,18782,18792,18802,18813,18823,18837,18843,18858,18863,18874,18887,18900,18911,18920,18932,18940,18950,18960,18984,18996,19024,19044,19129,19139,19149,19158,19168,19179,19193,19199,19210,19219,19229,19239,19249,19263,19270,19280,19290,19301,19310,19323,19331,19358,19360,19362,19377,19385,19395,19405,19415,19425,19436,19446,19460,19466,19478,19491,19496,19506,19517,19527,19537,19550,19557,19567,19577,19587,19606,19610,19617,19628,19638,19648,19660,19670,19681,19691,19701,19711,19721,19732,19742,19752,19762,19772,19784,19803,19871,19941,19943], + // biome-ignore format: generated capture data + sizes: [219,212,105,223,211,112,318,104,113,106,110,107,105,536,213,111,106,114,105,111,105,112,108,104,105,103,103,106,323,209,219,103,108,431,215,113,105,109,106,106,109,214,105,220,113,107,108,211,536,106,106,321,110,106,106,211,109,208,416,103,105,218,212,212,105,108,104,215,221,109,209,107,222,103,320,221,107,108,106,430,222,218,107,209,112,219,105,319,105,106,108,319,211,107,105,316,211,111,216,105,315,107,212,212,105,415,106,105,207,104,532,106,106,336,207,208,328,104,206,103,107,217,104,208,210,327,106,105,107,106,323,318,108,215,213,530,107,211,114,316,106,221,104,215,219,212,109,211,210,322,107,209,105,103,105,104,216,213,211,104,326,104,312,106,103,209,103,104,108,103,210,108,315,105,108,325,103,215,213,210,108,429,213,111,103,328,108,213,418,103,105,106,103,422,105,105,211,103,103,105,104,105,106,207,313,210,103,105,208,210,105,106,214,108,209,109,326,104,319,209,218,105,111,113,640,317,216,215,103,106,436,206,103,215,104,104,104,223,215,107,220,106,318,104,215,105,106,321,217,105,218,221,109,210,107,213,112,211,112,218,218,224,211,111,106,322,318,103,108,212,215,104,104,313,105,322,211,110,328,103,213,210,107,328,105,426,215,209,212,104,209,109,216,114,317,104,217,107,319,108,103,209,217,215,107,104,313,103,106,109,215,209,214,221,105,105,103,423,331,105,325,210,212,106,214,103,223,222,217,105,210,214,210,105,105,211,107,218,213,112,109,106,215,328,208,220,105,210,220,108,103,112,106,320,322,105,225,104,208,104,103,105,104,209,106,211,220,105,210,207,105,209,105,210,325,105,106,108,108,317,419,103,215,108,210,104,213,109,105,329,106,212,216,106,108,313,210,106,110,106,215,207,104,209,105,213,105,315,216,105,537,105,214,104,216,219,105,104,113,217,208,317,105,104,107,213,328,103,104,221,103,105,216,105,316,212,315,103,105,108,211,208,214,105,209,112,314,103,105,105,208,318,103,208,103,105,213,106,103,103,208,106,206,105,209,105,106,103,106,106,106,105,313,528,425,103,103,214,222,106,103,103,106,212,106,109,111,105,104,111,109,106,108,106,110,103,220,105,215,216,208,326,111,323,213,220,217,219,219,216,310,210,216,318,218,209,211,106,325,104,210,322,211,208,106,652,212,317,208,207,322,218,104,222,107,317,105,209,216,208,212,218,207,103,432,106,530,429,323,1062,1383,1180,1049,1398,1060,1064,106,212,110,211,112,209,215,218,104,221,106,217,105,214,966,419,107,220,208,103,322,220,105,315,103,320,421,107,329,211,211,319,323,111,213,223,113,223,320,206,103,107,214,219,105,210,108,210,107,217,213,105,217,216,217,105,210,110,213,214,214,108,216,105,217,103,106,110,113,216,221,105,319,103,106,112,105,424,105,105,209,104,313,206,209,104,105,103,105,208,315,104,104,226,212,106,210,110,219,211,208,103,109,107,214,106,217,107,225,106,207,213,108,211,105,218,216,105,328,103,106,211,217,104,217,107,217,103,222,221,103,105,106,108,218,209,104,208,106,209,104,206,315,104,103,209,209,210,103,103,208,208,217,103,103,220,210,106,213,105,214,325,103,213,108,112,106,103,112,105,104,105,104,103,210,215,109,212,111,319,103,107,211,116,213,110,311,104,111,321,104,103,209,209,106,209,107,218,212,105,333,105,106,215,104,220,209,106,212,211,106,216,216,113,113,209,113,106,108,212,105,107,217,217,103,218,108,105,216,212,106,212,107,213,214,105,213,325,109,104,206,103,110,216,214,107,220,214,103,223,213,106,213,105,217,313,105,320,104,104,105,111,111,108,104,105,103,106,103,110,311,211,108,108,106,210,324,108,315,103,105,424,104,105,105,424,104,104,105,209,104,209,309,104,214,214,210,104,103,209,103,420,104,110,423,104,105,105,103,112,106,103,419,105,216,209,106,324,210,105,106,111,318,210,222,107,110,212,225,209,103,219,215,105,209,216,111,215,108,213,219,111,223,214,216,320,103,103,209,103,209,110,221,109,325,104,310,104,212,111,212,105,107,318,106,325,103,210,107,216,209,217,104,216,109,213,208,106,106,209,213,216,106,315,105,108,212,216,220,103,209,105,215,216,107,213,212,206,206,103,105,105,104,106,105,310,108,210,212,103,105,209,211,108,209,217,220,110,212,208,103,225,107,217,208,209,213,111,211,212,105,214,210,104,218,107,104,319,209,211,216,109,217,217,109,213,104,214,107,103,222,113,221,216,331,213,206,103,111,220,208,216,214,215,213,106,315,103,210,418,105,219,104,209,209,211,214,103,211,105,330,103,219,216,314,209,104,104,103,209,103,104,522,104,208,209,105,217,207,210,109,210,209,217,105,318,217,103,104,217,216,219,106,207,210,110,105,213,209,104,218,316,103,214,210,109,215,215,103,314,209,103,215,209,219,104,211,209,107,312,209,103,315,207,207,103,103,106,107,316,103,326,213,221,208,216,108,424,104,103,209,104,315,103,207,208,206,104,206,104,206,106,213,224,109,215,216,106,215,213,217,106,210,316,103,225,209,210,210,110,208,219,209,213,219,106,222,216,218,217,212,113,323,103,219,209,213,103,314,211,103,211,106,208,312,104,208,310,104,310,103,104,213,210,219,106,224,211,209,218,109,211,330,106,218,216,105,211,103,223,315,106,217,221,216,106,322,103,214,218,106,419,104,103,209,104,105,210,103,103,213,316,103,107,218,216,107,216,103,218,209,107,214,211,207,212,211,106,221,108,104,103,109,107,106,222,109,318,103,319,104,208,311,105,211,219,104,521,104,105,103,210,311,103,109,107,111,104,215,207,208,108,311,211,103,415,105,103,221,111,209,104,208,415,103,107,215,107,325,103,212,214,107,213,213,214,215,107,209,105,104,315,325,104,218,215,104,112,109,218,111,214,220,216,106,107,108,639,103,108,212,222,103,208,104,106,104,104,106,109,107,107,106,112,107,107,105,217,109,325,110,215,219,219,105,217,220,112,109,109,216,105,312,104,214,209,112,320,104,415,105,213,113,110,112,105,108,107,105,213,103,430,104,320,105,103,217,316,108,213,221,210,103,212,103,105,420,104,109,222,212,106,103,105,103,104,109,114,105,323,104,110,220,216,215,104,215,216,215,107,321,103,212,112,326,103,208,104,321,224,315,105,208,105,106,103,103,105,310,105,105,209,311,207,208,104,313,210,103,108,313,210,103,105,106,109,212,217,213,313,103,105,109,211,108,217,218,108,321,107,103,104,324,107,219,111,214,207,209,215,221,109,213,221,313,104,107,416,220,213,209,112,107,213,104,207,207,213,214,212,105,221,217,211,211,113,107,318,207,104,216,319,309,105,103,213,112,104,104,105,221,107,211,315,309,105,311,104,104,108,328,214,218,212,103,215,111,216,211,219,218,103,208,213,108,217,215,112,107,216,216,109,210,109,217,213,107,212,213,207,104,211,216,107,329,103,217,105,322,106,214,105,218,103,108,530,105,103,211,103,103,107,210,105,108,211,209,107,215,211,207,211,103,316,105,108,211,316,103,105,107,211,103,314,105,318,103,106,215,313,103,208,109,219,220,211,104,218,108,319,110,219,218,107,216,223,318,104,107,108,106,113,104,103,104,104,112,105,226,104,214,113,212,221,115,107,108,113,320,110,209,329,629,332,1166,422,216,321,105,208,106,312,105,322,106,104,215,105,321,109,326,106,208,216,219,208,105,107,218,208,211,212,105,214,106,327,110,211,104,312,211,111,318,104,212,212,111,211,114,107,107,105,104,212,215,113,103,107,110,105,113,217,319,218,105,207,207,214,222,107,112,218,327,210,434,208,109,218,207,105,103,208,104,107,106,215,212,316,108,649,545,646,416,755,427,219,109,319,216,109,211,213,207,213,110,104,329,104,208,209,104,417,103,213,215,211,106,213,209,208,218,103,216,107,209,103,210,1537,6], + }, +}; diff --git a/workbench/example/workflows/97_bench_rtt.ts b/workbench/example/workflows/97_bench_rtt.ts new file mode 100644 index 0000000000..abcf8c490e --- /dev/null +++ b/workbench/example/workflows/97_bench_rtt.ts @@ -0,0 +1,390 @@ +// Pure bucketing + aggregation helpers for the chunk round-trip-time (CRTT) +// benchmark scenario. The workflow half lives in 97_bench.ts +// (benchCrttWorkflow) and the runner half in +// packages/core/e2e/benchmark.test.ts. +// +// This module is deliberately dependency-free so the same code runs in three +// places: the reader step aggregates per-chunk RTT samples on the deployment +// (keeping the workflow return value small — bucketed summaries, not hundreds +// of raw samples), the benchmark runner merges the per-iteration summaries +// into one row per bucket, and the unit tests +// (packages/core/src/bench-chunk-rtt-stats.test.ts) exercise both directly. + +/** + * Summary of one bucket's RTT samples (all values in ms, rounded to 0.1ms). + * Computed inside the reader step per iteration (exact percentiles over that + * iteration's samples), then merged across iterations by + * {@link mergeRttSummaries}. + */ +export interface BenchRttSummary { + /** Number of samples aggregated into this summary. */ + count: number; + /** Fastest sample (min). */ + best: number; + /** Mean — the exit criteria's headline "average per-chunk RTT". */ + avg: number; + p50: number; + p75: number; + p90: number; + p99: number; + /** Fixed-bin histogram of the samples (see {@link RTT_HIST_EDGES_MS}): + * `hist[i]` counts samples in `[edges[i-1], edges[i])`, with `hist[0]` + * below the first edge and the last entry at/above the last edge. Because + * the edges are a shared constant, histograms merge exactly — across + * iterations and across benchmark runs — unlike the percentile fields. */ + hist: number[]; +} + +// Histogram bin edges (ms), a 1-2-5 log series. Log-scale bins keep +// resolution at both ends of the plausible range — a warm in-region +// write->read can be single-digit ms while a stalled delivery is over a +// second — and fixed shared edges are what make cross-run histogram diffs +// exact (adaptive widths, like the STSO section's, cannot be re-binned once +// the raw samples have been left behind on the deployment). +export const RTT_HIST_EDGES_MS = [ + 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, +]; + +/** Buckets samples into the fixed {@link RTT_HIST_EDGES_MS} bins. Returns + * `edges.length + 1` counts (last = at/above the final edge). */ +export function histogramRttSamples(samples: number[]): number[] { + const counts = new Array(RTT_HIST_EDGES_MS.length + 1).fill(0); + for (const v of samples) { + let bin = 0; + while (bin < RTT_HIST_EDGES_MS.length && v >= RTT_HIST_EDGES_MS[bin]) { + bin++; + } + counts[bin]++; + } + return counts; +} + +// Chunk-index buckets. Each boundary is tied to a mechanism, not a progress +// range: +// - 'seq 0': the stream-open write (stream creation / cold write path). Also +// a cross-check against the SL scenario, which times the same first-chunk +// propagation. +// - 'seq 1-20': warmup — the first ~200ms at the modeled 100 chunks/s, where +// connections, buffers, and flush cycles are still settling. +// - 'seq 21+': steady state, kept as ONE bucket so its large n gives stable +// tail percentiles (splitting it further just compares noise floors of +// unequal sample sizes — iteration-level stalls land in whichever range +// they land in). +// Latency *drift* across the stream (cumulative log/buffer growth) is a +// trend, which fixed buckets detect badly; that is the progress profile's +// job (see {@link progressProfile}). +export const RTT_INDEX_BUCKETS = ['seq 0', 'seq 1-20', 'seq 21+'] as const; +export type RttIndexBucket = (typeof RTT_INDEX_BUCKETS)[number]; + +export function rttIndexBucket(seq: number): RttIndexBucket { + if (seq <= 0) return 'seq 0'; + if (seq <= 20) return 'seq 1-20'; + return 'seq 21+'; +} + +// Number of equal fractions of the stream in the progress profile. Ten keeps +// the profile line compact while still localizing a drift or a slow phase. +export const RTT_PROGRESS_BINS = 10; + +/** A binned mean-RTT profile: `totalMs[i]`/`counts[i]` is the mean RTT of + * bin i. Used for both the stream-progress profile (bin = tenth of the + * stream) and the chunk-size profile (bin = log size range). Sums and counts + * merge exactly across iterations and runs. */ +export interface BenchRttMeanProfile { + counts: number[]; + totalMs: number[]; +} + +/** Builds the progress profile from per-seq RTT samples (`rttBySeq[seq]` = + * that chunk's RTT; sparse entries are skipped defensively). Fraction-based + * (not absolute seq), so profiles are comparable across chunk counts. The + * trend this surfaces — does per-chunk RTT rise as the stream grows? — is + * what fixed index buckets cannot answer without arbitrary boundaries. */ +export function progressProfile( + rttBySeq: readonly (number | undefined)[] +): BenchRttMeanProfile { + const counts = new Array(RTT_PROGRESS_BINS).fill(0); + const totalMs = new Array(RTT_PROGRESS_BINS).fill(0); + const n = rttBySeq.length; + for (let seq = 0; seq < n; seq++) { + const rtt = rttBySeq[seq]; + if (typeof rtt !== 'number') continue; + const bin = Math.min( + RTT_PROGRESS_BINS - 1, + Math.floor((seq * RTT_PROGRESS_BINS) / n) + ); + counts[bin]++; + totalMs[bin] += rtt; + } + return { counts, totalMs }; +} + +/** Tail summary of a delay-style sample set where the MAX is the headline + * (one bad event among hundreds vanishes into pooled percentiles but is, by + * construction, the max). Used for write slip (producer-side lateness vs the + * open-loop schedule) and for positive CDV (delivery clumps/stalls). */ +export interface BenchDelayTail { + count: number; + avgMs: number; + p99Ms: number; + maxMs: number; +} + +/** Summarizes delay samples into a {@link BenchDelayTail}. */ +export function summarizeDelayTail( + samples: number[] +): BenchDelayTail | undefined { + if (samples.length === 0) return undefined; + const sorted = [...samples].sort((a, b) => a - b); + return { + count: sorted.length, + avgMs: round(sorted.reduce((sum, v) => sum + v, 0) / sorted.length), + p99Ms: round(percentile(sorted, 99)), + maxMs: round(sorted[sorted.length - 1]), + }; +} + +/** Sustained throughput over the steady window of a run: the first and last + * `trimFraction` of points (by index) are dropped so warmup (first-delivery + * setup) and drain (final flush) don't flatter or damn the sustained rate. */ +export interface BenchSteadyRate { + chunksPerSec: number; + kibPerSec: number; + /** Points inside the steady window. */ + windowChunks: number; + /** Wall span of the steady window (ms). */ + windowMs: number; +} + +/** Computes the steady-window rate from per-chunk (timestamp, bytes) points + * in stream order. Returns undefined when the window is too small to define + * a rate (fewer than 2 points or zero span). */ +export function steadyRate( + points: readonly { atMs: number; bytes: number }[], + trimFraction = 0.1 +): BenchSteadyRate | undefined { + const trim = Math.floor(points.length * trimFraction); + const window = points.slice(trim, points.length - trim); + if (window.length < 2) return undefined; + const spanMs = window[window.length - 1].atMs - window[0].atMs; + if (spanMs <= 0) return undefined; + // Both rates count events over the window's intervals: n points span n-1 + // gaps, and the first point's bytes "arrived" before the window's clock + // started — counting them would inflate a perfectly steady stream's + // byte rate by 1/(n-1). + const bytes = window.slice(1).reduce((sum, p) => sum + p.bytes, 0); + const round = (v: number) => Math.round(v * 10) / 10; + return { + chunksPerSec: round(((window.length - 1) * 1000) / spanMs), + kibPerSec: round((bytes * 1000) / spanMs / 1024), + windowChunks: window.length, + windowMs: spanMs, + }; +} + +/** One received chunk's RAW timestamps, in arrival order. CDV must be + * computed from unclamped values: clamping breaks the telescoping identity + * and hides the negative (catch-up) half of every delivery clump. */ +export interface CdvArrival { + seq: number; + writtenAt: number; + readAt: number; +} + +export interface BenchCdvComputation { + /** Signed cdv per seq-adjacent arrival pair, in arrival order. */ + cdvMs: number[]; + /** Positive cdv indexed by the later chunk's seq — progressProfile input + * (length is padded to max seq + 1 so fraction bins line up). */ + positiveBySeq: (number | undefined)[]; + duplicateSeqs: number; + reorderedArrivals: number; + /** Adjacent arrivals skipped because their seqs weren't consecutive. */ + skippedPairs: number; +} + +/** + * Chunk delay variation (delivery jitter): for seq-adjacent chunks received + * back to back, cdv_i = (readAt_i - readAt_{i-1}) - (writtenAt_i - + * writtenAt_{i-1}) = CTT_i - CTT_{i-1}. Each gap subtracts same-clock + * stamps, so CDV is skew-free — measurable in production where cross-clock + * CTT is not. Signed: clumped delivery of a 10ms-paced stream reads + * (-10, -10, +20); the sum telescopes, so report the positive tail, not + * means. Writer pauses self-exclude (both gaps grow equally). Pairs form + * only for chunks adjacent in BOTH arrival order and seq (pairing + * seq-sorted samples under reordering would manufacture phantom cdv); + * duplicates/reorders/holes are counted for the caller to treat as + * integrity failures. + */ +export function computeCdv( + arrivals: readonly CdvArrival[] +): BenchCdvComputation { + const seen = new Set(); + const cdvMs: number[] = []; + const positiveBySeq: (number | undefined)[] = []; + let duplicateSeqs = 0; + let reorderedArrivals = 0; + let skippedPairs = 0; + let maxSeq = -1; + for (let i = 0; i < arrivals.length; i++) { + const chunk = arrivals[i]; + if (seen.has(chunk.seq)) duplicateSeqs++; + seen.add(chunk.seq); + if (chunk.seq > maxSeq) maxSeq = chunk.seq; + if (i === 0) continue; // the first arrival anchors; it has no pair + const prev = arrivals[i - 1]; + if (chunk.seq < prev.seq) reorderedArrivals++; + if (chunk.seq !== prev.seq + 1) { + skippedPairs++; + continue; + } + const cdv = chunk.readAt - prev.readAt - (chunk.writtenAt - prev.writtenAt); + cdvMs.push(cdv); + if (cdv > 0) positiveBySeq[chunk.seq] = cdv; + } + positiveBySeq.length = Math.max(positiveBySeq.length, maxSeq + 1); + return { + cdvMs, + positiveBySeq, + duplicateSeqs, + reorderedArrivals, + skippedPairs, + }; +} + +/** Merges mean profiles by summation — exact, like the histograms. */ +export function mergeMeanProfiles( + profiles: readonly (BenchRttMeanProfile | undefined)[] +): BenchRttMeanProfile | undefined { + const present = profiles.filter((p): p is BenchRttMeanProfile => p != null); + if (present.length === 0) return undefined; + const bins = Math.max(...present.map((p) => p.counts.length)); + const counts = new Array(bins).fill(0); + const totalMs = new Array(bins).fill(0); + for (const p of present) { + for (let i = 0; i < bins; i++) { + counts[i] += p.counts[i] ?? 0; + totalMs[i] += p.totalMs[i] ?? 0; + } + } + return { counts, totalMs }; +} + +// Chunk-size profile bins (approximate serialized bytes, doubling edges). +// Bin i covers [edges[i-1], edges[i]), bin 0 everything below 256B, and the +// last bin everything at/above 8KB. The size-sweep scenario's pad rotation +// (see CRTT_SWEEP_PAD_LENGTHS in 97_bench.ts) puts one padded size in each +// bin, so the mean-RTT-per-bin profile is a size→latency curve: flat means +// chunk size doesn't matter, a knee localizes where it starts to. +export const RTT_SIZE_BIN_EDGES_BYTES = [256, 512, 1024, 2048, 4096, 8192]; + +/** Bin index into {@link RTT_SIZE_BIN_EDGES_BYTES} for a serialized size. */ +export function rttSizeBin(serializedBytes: number): number { + let bin = 0; + while ( + bin < RTT_SIZE_BIN_EDGES_BYTES.length && + serializedBytes >= RTT_SIZE_BIN_EDGES_BYTES[bin] + ) { + bin++; + } + return bin; +} + +/** Builds the chunk-size profile from (serialized bytes, RTT) samples. */ +export function sizeProfile( + samples: readonly { bytes: number; rttMs: number }[] +): BenchRttMeanProfile { + const bins = RTT_SIZE_BIN_EDGES_BYTES.length + 1; + const counts = new Array(bins).fill(0); + const totalMs = new Array(bins).fill(0); + for (const { bytes, rttMs } of samples) { + const bin = rttSizeBin(bytes); + counts[bin]++; + totalMs[bin] += rttMs; + } + return { counts, totalMs }; +} + +// Same percentile convention as the benchmark runner's computeStats +// (nearest-rank via ceil), so a CRTT p90 means the same thing as an SO p90. +function percentile(sortedAscending: number[], q: number): number { + return sortedAscending[ + Math.min( + sortedAscending.length - 1, + Math.ceil((q / 100) * sortedAscending.length) - 1 + ) + ]; +} + +const round = (v: number) => Math.round(v * 10) / 10; + +/** Exact summary of one iteration's samples for a bucket; undefined when the + * bucket received no samples (so the caller can just skip it). */ +export function summarizeRttSamples( + samples: number[] +): BenchRttSummary | undefined { + if (samples.length === 0) return undefined; + const sorted = [...samples].sort((a, b) => a - b); + return { + count: sorted.length, + best: round(sorted[0]), + avg: round(sorted.reduce((sum, v) => sum + v, 0) / sorted.length), + hist: histogramRttSamples(sorted), + p50: round(percentile(sorted, 50)), + p75: round(percentile(sorted, 75)), + p90: round(percentile(sorted, 90)), + p99: round(percentile(sorted, 99)), + }; +} + +/** + * Merges per-iteration bucket summaries. count/best/avg (count-weighted) and + * hist (elementwise over shared fixed bins) are exact; p50-p99 are + * percentile-of-percentiles (raw samples never leave the reader step) — + * exact only for single-sample iterations (e.g. seq 0), an approximation + * for headline rows. Good enough for trend tracking; the histogram is the + * exact pooled view. + */ +export function mergeRttSummaries( + summaries: readonly (BenchRttSummary | undefined)[] +): BenchRttSummary | undefined { + const present = summaries.filter((s): s is BenchRttSummary => s != null); + if (present.length === 0) return undefined; + const count = present.reduce((sum, s) => sum + s.count, 0); + const mergedPercentile = (q: number, values: number[]) => + round( + percentile( + [...values].sort((a, b) => a - b), + q + ) + ); + const histLength = Math.max(...present.map((s) => s.hist?.length ?? 0)); + const hist = new Array(histLength).fill(0); + for (const s of present) { + (s.hist ?? []).forEach((c, i) => { + hist[i] += c; + }); + } + return { + count, + best: round(Math.min(...present.map((s) => s.best))), + avg: round(present.reduce((sum, s) => sum + s.avg * s.count, 0) / count), + hist, + p50: mergedPercentile( + 50, + present.map((s) => s.p50) + ), + p75: mergedPercentile( + 75, + present.map((s) => s.p75) + ), + p90: mergedPercentile( + 90, + present.map((s) => s.p90) + ), + p99: mergedPercentile( + 99, + present.map((s) => s.p99) + ), + }; +} diff --git a/workbench/nextjs-turbopack/workflows/97_bench_cadence.ts b/workbench/nextjs-turbopack/workflows/97_bench_cadence.ts new file mode 120000 index 0000000000..4c9bfa4d0e --- /dev/null +++ b/workbench/nextjs-turbopack/workflows/97_bench_cadence.ts @@ -0,0 +1 @@ +../../example/workflows/97_bench_cadence.ts \ No newline at end of file diff --git a/workbench/nextjs-turbopack/workflows/97_bench_rtt.ts b/workbench/nextjs-turbopack/workflows/97_bench_rtt.ts new file mode 120000 index 0000000000..2463f17ab7 --- /dev/null +++ b/workbench/nextjs-turbopack/workflows/97_bench_rtt.ts @@ -0,0 +1 @@ +../../example/workflows/97_bench_rtt.ts \ No newline at end of file diff --git a/workbench/nitro-v3/workflows/97_bench_cadence.ts b/workbench/nitro-v3/workflows/97_bench_cadence.ts new file mode 120000 index 0000000000..4c9bfa4d0e --- /dev/null +++ b/workbench/nitro-v3/workflows/97_bench_cadence.ts @@ -0,0 +1 @@ +../../example/workflows/97_bench_cadence.ts \ No newline at end of file diff --git a/workbench/nitro-v3/workflows/97_bench_rtt.ts b/workbench/nitro-v3/workflows/97_bench_rtt.ts new file mode 120000 index 0000000000..2463f17ab7 --- /dev/null +++ b/workbench/nitro-v3/workflows/97_bench_rtt.ts @@ -0,0 +1 @@ +../../example/workflows/97_bench_rtt.ts \ No newline at end of file From db1ef6627f9bae705c7ce2a85c3cc9979ad2b41c Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Thu, 13 Aug 2026 17:36:59 -0500 Subject: [PATCH 2/3] bench: round medians and table cells - float artifacts broke the layout medianOf's even-count average produces values like 54.650000000000006, which rendered verbatim and blew out the stream table's column widths. Round the median to two decimals at the source (it lands in artifacts and history too) and round sub-100 values to one decimal in formatMs as the display-side guard. Signed-off-by: Alex Langenfeld --- .github/scripts/render-benchmark-comment.mjs | 4 +++- packages/core/e2e/benchmark.test.ts | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/scripts/render-benchmark-comment.mjs b/.github/scripts/render-benchmark-comment.mjs index eb25f7f588..24ee66f42c 100644 --- a/.github/scripts/render-benchmark-comment.mjs +++ b/.github/scripts/render-benchmark-comment.mjs @@ -248,7 +248,9 @@ export function loadResults(resultsDir) { function formatMs(value) { if (typeof value !== 'number' || !Number.isFinite(value)) return '—'; - return `${Math.abs(value) >= 100 ? Math.round(value) : value}`; + // Round to one decimal below 100 (and trim float artifacts like + // 54.650000000000006 from upstream averaging), integers above. + return `${Math.abs(value) >= 100 ? Math.round(value) : Math.round(value * 10) / 10}`; } /** diff --git a/packages/core/e2e/benchmark.test.ts b/packages/core/e2e/benchmark.test.ts index 7b59719cc0..5ef838f315 100644 --- a/packages/core/e2e/benchmark.test.ts +++ b/packages/core/e2e/benchmark.test.ts @@ -619,9 +619,13 @@ function medianOf(values: readonly (number | undefined)[]): number | undefined { if (present.length === 0) return undefined; const sorted = [...present].sort((a, b) => a - b); const mid = sorted.length / 2; - return sorted.length % 2 === 1 - ? sorted[Math.floor(mid)] - : (sorted[mid - 1] + sorted[mid]) / 2; + const median = + sorted.length % 2 === 1 + ? sorted[Math.floor(mid)] + : (sorted[mid - 1] + sorted[mid]) / 2; + // Inputs carry one decimal; averaging two of them yields float artifacts + // (54.650000000000006) that leak into the table and artifacts unrounded. + return Math.round(median * 100) / 100; } /** From d75dfb9aacd18ff1e2f0752c5a28ff602cd94b20 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Thu, 13 Aug 2026 20:12:29 -0500 Subject: [PATCH 3/3] changeset for the bench scenarios (empty - no package bumps) Signed-off-by: Alex Langenfeld --- .changeset/bench-chunk-rtt-scenarios.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .changeset/bench-chunk-rtt-scenarios.md diff --git a/.changeset/bench-chunk-rtt-scenarios.md b/.changeset/bench-chunk-rtt-scenarios.md new file mode 100644 index 0000000000..3052b01fbc --- /dev/null +++ b/.changeset/bench-chunk-rtt-scenarios.md @@ -0,0 +1,4 @@ +--- +--- + +Add per-chunk stream latency to the CI benchmark: CRTT/CDV metrics with a paced control, a size sweep, and replay scenarios driven by real captured cadences at the eve and AI-gateway boundaries, reported in a dedicated Streams table plus a pooled first-chunk RTT row; the SL/SO report rows are retired (CRTT subsumes both).