diff --git a/.changeset/bench-chunk-rtt-scenarios.md b/.changeset/bench-chunk-rtt-scenarios.md
new file mode 100644
index 0000000000..3052b01fbc
--- /dev/null
+++ b/.changeset/bench-chunk-rtt-scenarios.md
@@ -0,0 +1,4 @@
+---
+---
+
+Add per-chunk stream latency to the CI benchmark: CRTT/CDV metrics with a paced control, a size sweep, and replay scenarios driven by real captured cadences at the eve and AI-gateway boundaries, reported in a dedicated Streams table plus a pooled first-chunk RTT row; the SL/SO report rows are retired (CRTT subsumes both).
diff --git a/.github/scripts/render-benchmark-comment.mjs b/.github/scripts/render-benchmark-comment.mjs
index 51af3ec75d..24ee66f42c 100644
--- a/.github/scripts/render-benchmark-comment.mjs
+++ b/.github/scripts/render-benchmark-comment.mjs
@@ -31,7 +31,7 @@ const METRIC_LABELS = {
ttfs: {
name: 'TTFS',
description:
- 'time to first step body (in-deployment start() → first step body, deployment clocks)',
+ 'time to first step body (in-deployment start() → first step body)',
},
'fanout-ttfs': {
name: 'Fan-out TTFS',
@@ -62,6 +62,26 @@ const METRIC_LABELS = {
description:
'stream overhead (end-to-end write+consume time beyond the modelled generation window)',
},
+ // Name reservations: CTT = future production one-way write→read metric
+ // (cross-clock); TTFC = future consumer-journey start → first-chunk-readable
+ // metric. The 'first chunk (pooled)' row is neither (readAt₀ - writtenAt₀,
+ // a round trip), so it stays under CRTT.
+ crtt: {
+ name: 'CRTT',
+ description:
+ 'chunk round-trip time (per-chunk write → read latency, one clock domain: deployment → stream backend → same deployment)',
+ },
+ cdv: {
+ name: 'CDV',
+ description:
+ "chunk delay variation / delivery jitter (inter-arrival gap minus inter-write gap per seq-adjacent pair; skew-free; the row is each run's MAX positive value, so one stall moves it)",
+ },
+ slip: {
+ // Title-case on purpose (a word, not an initialism). Artifact-only.
+ name: 'Slip',
+ description:
+ "write slip (how late each chunk was written vs the writer's open-loop schedule; the row is each run's MAX — the producer-stall guard that RTT and CDV both hide)",
+ },
};
const METRIC_ORDER = [
'ttfs',
@@ -71,6 +91,9 @@ const METRIC_ORDER = [
'wo',
'sl',
'so',
+ 'crtt',
+ 'cdv',
+ 'slip',
];
export function parseArgs(argv) {
@@ -138,17 +161,19 @@ export function extractHistory(body) {
}
/**
- * Drops the per-metric raw sample arrays before embedding an entry in the
- * comment's data block. The sequential-steps scenario records ~1000 STSO
- * samples per run (plus the baseline's), which would blow past GitHub's
- * comment size limit within a couple of history entries; the percentiles and
- * baseline annotations — everything the history tables render — are kept.
+ * Drops the per-metric raw sample arrays (and the CRTT fixed-bin histograms)
+ * before embedding an entry in the comment's data block. The sequential-steps
+ * scenario records ~1000 STSO samples per run (plus the baseline's), which
+ * would blow past GitHub's comment size limit within a couple of history
+ * entries; the percentiles and baseline annotations — everything the history
+ * tables render — are kept.
*
- * This does not affect the histogram diff against `main`: that reads its
- * baseline from the artifacts the workflow downloads into --baseline-dir,
- * which keep their raw samples. What it costs is the collapsed "Previous
- * results" entries, re-rendered from this block on a later commit of the same
- * PR — they show their tables but not their histograms.
+ * This does not affect the distribution diffs against `main`: those read
+ * their baselines from the artifacts the workflow downloads into
+ * --baseline-dir, which keep raw samples and histograms. What it costs is the
+ * collapsed "Previous results" entries, re-rendered from this block on a
+ * later commit of the same PR — they show their tables but not their
+ * histograms.
*/
function stripRawSamples(entries) {
return entries.map((entry) => ({
@@ -156,7 +181,23 @@ function stripRawSamples(entries) {
results: (entry.results ?? []).map((result) => ({
...result,
metrics: (result.metrics ?? []).map(
- ({ raw, baselineRaw, ...row }) => row
+ ({
+ raw,
+ baselineRaw,
+ hist,
+ progressAvgMs,
+ sizeAvgMs,
+ cdvAvgMs,
+ ...row
+ }) => {
+ // Stream rows: keep the median columns for history tables, drop
+ // the per-run arrays.
+ if (row.stream?.runs) {
+ const { runs, ...medians } = row.stream;
+ return { ...row, stream: medians };
+ }
+ return row;
+ }
),
})),
}));
@@ -207,7 +248,9 @@ export function loadResults(resultsDir) {
function formatMs(value) {
if (typeof value !== 'number' || !Number.isFinite(value)) return '—';
- return `${Math.abs(value) >= 100 ? Math.round(value) : value}`;
+ // Round to one decimal below 100 (and trim float artifacts like
+ // 54.650000000000006 from upstream averaging), integers above.
+ return `${Math.abs(value) >= 100 ? Math.round(value) : Math.round(value * 10) / 10}`;
}
/**
@@ -228,6 +271,10 @@ const BASELINE_FIELDS = [
{ annotation: 'baselineP75', from: (base) => base.p75 },
{ annotation: 'baselineP90', from: (base) => base.p90 },
{ annotation: 'baselineP99', from: (base) => base.p99 },
+ // Not rendered in the main table (no Avg/P50 columns there), but the CRTT
+ // drill-down matrix shows vs-main deltas on both (avg deltas are exact).
+ { annotation: 'baselineAvg', from: (base) => base.avg },
+ { annotation: 'baselineP50', from: (base) => base.p50 },
];
export function annotateWithBaseline(results, baseline) {
@@ -253,6 +300,12 @@ export function annotateWithBaseline(results, baseline) {
// histogram diff below the table — kept separate from BASELINE_FIELDS
// since it's an array, not a numeric percentile.
if (Array.isArray(base.raw)) annotated.baselineRaw = base.raw;
+ // Stream rows diff their rate/CDV columns against the baseline's stream
+ // object (per-run arrays dropped — only the medians are compared).
+ if (row.stream && base.stream) {
+ const { runs, ...medians } = base.stream;
+ annotated.baselineStream = medians;
+ }
return annotated;
};
return results.map((result) => ({
@@ -389,8 +442,13 @@ function renderOverlayBar(base, cur, maxCount) {
* and their delta on the same line (a fenced code block keeps everything
* aligned in a monospace font). This is the whole histogram diff — the shape
* of the two distributions and the per-bucket numbers behind it, without a
- * second table restating them. */
-function renderStsoBarChart(buckets, { selfDiff } = {}) {
+ * second table restating them. Shared by the STSO section (buckets = step
+ * counts) and the CRTT section (buckets = chunk counts); `selfLabel` names
+ * the series when there is no baseline to overlay. */
+function renderHistogramBarChart(
+ buckets,
+ { selfDiff, selfLabel = 'steps' } = {}
+) {
const maxCount = Math.max(1, ...buckets.map((b) => Math.max(b.base, b.cur)));
const labelWidth = Math.max(...buckets.map((b) => b.label.length));
const countWidth = Math.max(
@@ -408,7 +466,7 @@ function renderStsoBarChart(buckets, { selfDiff } = {}) {
: renderOverlayBar(base, cur, maxCount)
).padEnd(BAR_CHART_WIDTH);
const counts = selfDiff
- ? `steps ${String(cur).padStart(countWidth)}`
+ ? `${selfLabel} ${String(cur).padStart(countWidth)}`
: `main ${String(base).padStart(countWidth)} this ${String(cur).padStart(countWidth)} ${formatDeltaValue(cur - base).padStart(countWidth + 1)}`;
lines.push(`${label.padStart(labelWidth)} ms ${bar} ${counts}`);
}
@@ -464,7 +522,7 @@ function renderStsoRowDiff(row) {
);
}
if (buckets.length > 0) {
- lines.push(renderStsoBarChart(buckets, { selfDiff }));
+ lines.push(renderHistogramBarChart(buckets, { selfDiff }));
}
return lines.join('\n');
}
@@ -498,6 +556,166 @@ function renderStsoDiffSection(result) {
].join('\n');
}
+// ============================================================================
+// CRTT drill-down (per-bucket sparkline matrix, vs main)
+// ============================================================================
+
+const SPARK_LEVELS = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
+
+/** One-character-per-bin sparkline over fixed histogram counts, normalized to
+ * the row's own max so every bucket's *shape* is readable regardless of its
+ * sample count. Empty bins render as `·` so the fixed log axis stays visible
+ * and the occupied bins' *position* on it (fast vs slow) is comparable across
+ * lines. */
+function sparkline(counts) {
+ const max = Math.max(1, ...counts);
+ return counts
+ .map((c) =>
+ c === 0
+ ? '·'
+ : SPARK_LEVELS[
+ Math.min(
+ SPARK_LEVELS.length - 1,
+ Math.floor((c / max) * SPARK_LEVELS.length)
+ )
+ ]
+ )
+ .join('');
+}
+
+/**
+ * Renders the CRTT drill-down: ONE line per variant — a sparkline of the
+ * fixed log-bin RTT histogram plus avg/p50/p90/p99 (plain vs-main
+ * percentages when a baseline exists) — followed by the mean-RTT profile
+ * lines. Per-index detail rows are deliberately NOT rendered: three runs
+ * showed them flat and their run-to-run flips are bucket-hopping iteration
+ * noise that invites misreads. They stay in the results JSON (with baseline
+ * annotations), so when a headline delta fires the artifact still localizes
+ * it; the progress line guards position-dependence here with finer
+ * resolution than the buckets did.
+ *
+ * The avg deltas are exact (count-weighted merges on both sides); p50-p99
+ * are cross-iteration percentile-of-percentiles, like the main table.
+ * Collapsed by default, like the STSO section: a drill-down, not the
+ * headline.
+ */
+function renderCrttMatrixSection(result) {
+ const rows = (result.metrics ?? []).filter(
+ (row) => row.stream && !row.detail && Array.isArray(row.hist?.counts)
+ );
+ if (rows.length === 0) return '';
+ const anyBaseline = rows.some((row) => typeof row.baselineAvg === 'number');
+
+ const round1 = (v) => Math.round(v * 10) / 10;
+ const pct = (cur, base) => {
+ if (typeof cur !== 'number' || typeof base !== 'number' || base <= 0) {
+ return '';
+ }
+ const p = ((cur - base) / base) * 100;
+ if (Math.abs(p) < 0.5) return ' (±0%)';
+ return ` (${p > 0 ? '+' : ''}${Math.round(p)}%)`;
+ };
+ const cells = (row) => [
+ row.group ?? row.scenario,
+ sparkline(row.hist.counts),
+ `${round1(row.avg)}${pct(row.avg, row.baselineAvg)}`,
+ `${formatMs(row.p50)}${pct(row.p50, row.baselineP50)}`,
+ `${formatMs(row.p90)}${pct(row.p90, row.baselineP90)}`,
+ `${formatMs(row.p99)}${pct(row.p99, row.baselineP99)}`,
+ String(row.samples),
+ ];
+ const header = ['variant', 'RTT 1ms→5s+', 'avg', 'p50', 'p90', 'p99', 'n'];
+ const table = [header, ...rows.map(cells)];
+ const widths = header.map((_, col) =>
+ Math.max(...table.map((line) => line[col].length))
+ );
+ const renderLine = (line) =>
+ line
+ .map((cell, col) =>
+ // Left-align the label and sparkline columns, right-align numbers.
+ col <= 1 ? cell.padEnd(widths[col]) : cell.padStart(widths[col])
+ )
+ .join(' ')
+ .trimEnd();
+
+ const lines = ['```', renderLine(header)];
+ for (const row of rows) {
+ lines.push(renderLine(cells(row)));
+ }
+ lines.push('```');
+
+ // Mean-RTT profile lines, one per variant that recorded the profile:
+ // - progress (per tenth of the stream): the drift readout — a rising
+ // staircase means chunks get slower as the stream grows, which fixed
+ // index buckets can't localize.
+ // - size (per log size bin, sweep only): the size→latency curve — flat
+ // means chunk size doesn't matter, a knee localizes where it starts to.
+ // Bars are scaled min→max per line so the *shape* stays readable even for
+ // small effects; the ms range alongside is what says whether the shape
+ // matters. Null entries (empty bins) render as `·`.
+ const renderProfileBlock = (title, entries) => {
+ if (entries.length === 0) return;
+ const labelWidth = Math.max(...entries.map((e) => e.label.length));
+ lines.push('', title, '', '```');
+ for (const { label, avgs } of entries) {
+ const present = avgs.filter((v) => typeof v === 'number');
+ if (present.length === 0) continue;
+ const min = Math.min(...present);
+ const max = Math.max(...present);
+ const span = max - min;
+ const bars = avgs
+ .map((v) =>
+ typeof v !== 'number'
+ ? '·'
+ : span <= 0
+ ? SPARK_LEVELS[0]
+ : SPARK_LEVELS[
+ Math.round(((v - min) / span) * (SPARK_LEVELS.length - 1))
+ ]
+ )
+ .join('');
+ const range = `${Math.round(min)}–${Math.round(max)}ms`;
+ lines.push(`${label.padEnd(labelWidth)} ${bars} ${range}`);
+ }
+ lines.push('```');
+ };
+ const profileEntries = (field) =>
+ rows
+ .filter((row) => Array.isArray(row[field]) && row[field].length > 0)
+ .map((row) => ({ label: row.group ?? row.scenario, avgs: row[field] }));
+ renderProfileBlock(
+ 'RTT over stream progress (avg per tenth of stream, bars scaled min→max):',
+ profileEntries('progressAvgMs')
+ );
+ renderProfileBlock(
+ 'RTT by chunk size (avg per log size bin, ~160B → ~12KB serialized, bars scaled min→max):',
+ profileEntries('sizeAvgMs')
+ );
+ // Where in the stream delivery clumping/stalls concentrate — the CDV
+ // row's per-run max says the worst stall's size; this says where. Flat is
+ // steady-cadence clumping; a hot spot localizes a stall.
+ renderProfileBlock(
+ 'Delivery jitter over stream progress (avg positive CDV per tenth of stream, bars scaled min→max):',
+ profileEntries('cdvAvgMs')
+ );
+
+ return [
+ '',
+ '',
+ `📈 CRTT drill-down${anyBaseline ? ' vs main' : ''} (RTT distributions & profiles)
`,
+ '',
+ ...(anyBaseline
+ ? []
+ : [
+ 'No `main` baseline yet — percentages appear once a run on `main` has recorded CRTT.',
+ '',
+ ]),
+ lines.join('\n'),
+ '',
+ ' ',
+ ].join('\n');
+}
+
// Deltas beyond ±this vs main get a directional marker: 🔻 for a regression,
// 💚 for an improvement. Smaller moves show the percentage alone.
const DELTA_MARK_THRESHOLD_PCT = 15;
@@ -542,19 +760,83 @@ function shortCommit(commit) {
return commit ? commit.slice(0, 7) : 'unknown';
}
+// CDV (and, in older history entries, Slip) is the companion of CRTT,
+// measured by the same scenario runs (same workload, same iterations). The
+// table pairs each variant's rows — chunk RTT (llm) directly above delivery
+// jitter (llm) — instead of grouping metric by metric.
+const PAIRED_METRICS = { cdv: 'crtt', slip: 'crtt' };
+
function metricSortKey(row) {
- const idx = METRIC_ORDER.indexOf(row.metric);
+ const idx = METRIC_ORDER.indexOf(PAIRED_METRICS[row.metric] ?? row.metric);
return idx === -1 ? METRIC_ORDER.length : idx;
}
+/** Orders rows within a paired-metric family: by variant (`group`, falling
+ * back to scenario for rows recorded before `group` existed), then anchor
+ * metric before companion. Non-family rows keep insertion order (0 preserves
+ * the stable sort). */
+function pairedSortKey(a, b) {
+ const inFamily = (metric) =>
+ metric in PAIRED_METRICS || Object.values(PAIRED_METRICS).includes(metric);
+ if (!inFamily(a.metric) || !inFamily(b.metric)) return 0;
+ return (
+ (a.group ?? a.scenario).localeCompare(b.group ?? b.scenario) ||
+ METRIC_ORDER.indexOf(a.metric) - METRIC_ORDER.indexOf(b.metric)
+ );
+}
+
+/**
+ * The stream-scenario table: one row per stream scenario, with the columns
+ * streams actually want — writer-side achieved and reader-side delivered
+ * sustained rates (steady window: first/last 10% of chunks trimmed), CRTT
+ * percentiles, and the median worst delivery stall (CDV max positive).
+ * Deltas vs main are plain percentages; deliberately NO 🔴/🟢 marks — targets
+ * attach in a later PR once a baseline exists. Rates read higher-is-better,
+ * latencies lower-is-better, so directional marks would need per-column
+ * polarity anyway; numbers + deltas keep it honest until then.
+ */
+function renderStreamTable(result) {
+ const rows = (result.metrics ?? []).filter(
+ (row) => row.stream && !row.detail
+ );
+ if (rows.length === 0) return '';
+ const pct = (cur, base) => {
+ if (typeof cur !== 'number' || typeof base !== 'number' || base <= 0) {
+ return '';
+ }
+ const p = ((cur - base) / base) * 100;
+ if (Math.abs(p) < 0.5) return ' (±0%)';
+ return ` (${p > 0 ? '+' : ''}${Math.round(p)}%)`;
+ };
+ const cell = (value, base) =>
+ typeof value === 'number' ? `${formatMs(value)}${pct(value, base)}` : '—';
+ const lines = [
+ '**Streams**',
+ '',
+ '| Scenario | wr c/s | rd c/s | wr KiB/s | rd KiB/s | CRTT 1st | p75 | p90 | p99 | CDV max | iters |',
+ '|----------|-------:|-------:|---------:|---------:|---------:|----:|----:|----:|--------:|------:|',
+ ];
+ for (const row of rows) {
+ const s = row.stream;
+ const b = row.baselineStream ?? {};
+ lines.push(
+ `| ${row.scenario} | ${cell(s.wrCps, b.wrCps)} | ${cell(s.rdCps, b.rdCps)} | ${cell(s.wrKiBps, b.wrKiBps)} | ${cell(s.rdKiBps, b.rdKiBps)} | ${cell(s.firstMs, b.firstMs)} | ${cell(row.p75, row.baselineP75)} | ${cell(row.p90, row.baselineP90)} | ${cell(row.p99, row.baselineP99)} | ${cell(s.cdvMaxMs, b.cdvMaxMs)} | ${s.iterations} |`
+ );
+ }
+ return lines.join('\n');
+}
+
function renderResultTable(result) {
const lines = [
'| Metric | Scenario | Best (ms) | P75 (ms) | P90 (ms) | P99 (ms) | Samples |',
'|--------|----------|----------:|---------:|---------:|---------:|--------:|',
];
- const rows = [...result.metrics].sort(
- (a, b) => metricSortKey(a) - metricSortKey(b)
- );
+ // Drill-down rows (e.g. CRTT's per-bucket splits) and stream rows (their
+ // own table) stay out of the headline table.
+ const rows = result.metrics
+ .filter((row) => !row.detail && !row.stream)
+ .sort((a, b) => metricSortKey(a) - metricSortKey(b) || pairedSortKey(a, b));
+ if (rows.length === 0) return '';
for (const row of rows) {
const label = METRIC_LABELS[row.metric];
// Abbreviations only — the definitions live in the comment footer.
@@ -586,13 +868,18 @@ function renderEntry(entry, { heading }) {
} else {
lines.push(`Backend: \`${result.backend}\` · app: \`${result.app}\``, '');
}
- lines.push(renderResultTable(result), '');
- // Only the latest entry carries raw samples (they are stripped before
- // being embedded in the comment's data block, see stripRawSamples), so
- // this renders for the current run and is silently skipped for the
- // collapsed history entries.
+ const resultTable = renderResultTable(result);
+ if (resultTable) lines.push(resultTable, '');
+ const streamTable = renderStreamTable(result);
+ if (streamTable) lines.push(streamTable, '');
+ // Only the latest entry carries raw samples and histograms (they are
+ // stripped before being embedded in the comment's data block, see
+ // stripRawSamples), so these render for the current run and are silently
+ // skipped for the collapsed history entries.
const stsoDiff = renderStsoDiffSection(result);
if (stsoDiff) lines.push(stsoDiff, '');
+ const crttDiff = renderCrttMatrixSection(result);
+ if (crttDiff) lines.push(crttDiff, '');
}
return lines.join('\n');
}
@@ -610,6 +897,25 @@ function buildScenarioLegend(results) {
.join(' · ');
}
+/**
+ * Replay-cadence identity line: capture id + full semantic sha256 (the
+ * cross-system workload fingerprint — durabench computes the same hash over
+ * its copy of the capture; see cadenceSemanticSha256 in benchmark.test.ts).
+ * Rendered as its own line so the full hash is findable and copyable rather
+ * than buried in the scenario prose.
+ */
+function buildCadencesLegend(results) {
+ const cadences = new Map();
+ for (const result of results) {
+ for (const c of result.config?.replayCadences ?? []) {
+ if (c?.id && c?.semanticSha256 && !cadences.has(c.id)) {
+ cadences.set(c.id, c.semanticSha256);
+ }
+ }
+ }
+ return [...cadences].map(([id, sha]) => `**${id}** \`${sha}\``).join(' · ');
+}
+
/** Targets legend, derived from the per-row targets in the results. */
function buildTargetsLegend(results) {
const targets = new Map();
@@ -630,10 +936,35 @@ function buildTargetsLegend(results) {
function renderFooter(entries) {
const results = entries.flatMap((entry) => entry.results ?? []);
- const definitions = METRIC_ORDER.map(
- (id) => `**${METRIC_LABELS[id].name}**: ${METRIC_LABELS[id].description}`
- ).join(' · ');
+ // Only define the metrics this comment actually shows — retired metrics
+ // (e.g. SL/SO, superseded by CRTT) stay defined in METRIC_LABELS so older
+ // history entries keep rendering, but they drop out of the legend once the
+ // latest run no longer reports them.
+ // Only rendered rows feed the legend — artifact-only detail rows (e.g.
+ // per-index CRTT splits, slip tails) don't define terms the comment never
+ // shows.
+ const presentMetrics = new Set(
+ results.flatMap((result) =>
+ (result.metrics ?? [])
+ .filter((row) => !row.detail)
+ .map((row) => row.metric)
+ )
+ );
+ // The stream table's columns are CRTT percentiles and CDV max, so those
+ // definitions stay in the legend whenever stream rows render even though
+ // no row carries those metric ids anymore.
+ if (results.some((result) => (result.metrics ?? []).some((r) => r.stream))) {
+ presentMetrics.add('crtt');
+ presentMetrics.add('cdv');
+ presentMetrics.delete('stream');
+ }
+ const definitions = METRIC_ORDER.filter((id) => presentMetrics.has(id))
+ .map(
+ (id) => `**${METRIC_LABELS[id].name}**: ${METRIC_LABELS[id].description}`
+ )
+ .join(' · ');
const scenarioLegend = buildScenarioLegend(results);
+ const cadencesLegend = buildCadencesLegend(results);
const targetsLegend = buildTargetsLegend(results);
const hasBaseline = results.some((result) =>
(result.metrics ?? []).some(
@@ -651,10 +982,32 @@ function renderFooter(entries) {
)
);
+ const hasCrttDistribution = results.some((result) =>
+ (result.metrics ?? []).some(
+ (row) => row.stream && Array.isArray(row.hist?.counts)
+ )
+ );
+
+ const hasStreamTable = results.some((result) =>
+ (result.metrics ?? []).some((row) => row.stream)
+ );
+
const smallprint = [
+ ...(hasStreamTable
+ ? [
+ '**Streams**: writer/reader sustained rates (steady window, 10% trimmed each side), first-chunk RTT (the stream-open path, before any buffering/backpressure), CRTT percentiles, and worst delivery stall (CDV max). Cells are medians across iterations; per-run values in the artifacts. No \ud83d\udd34/\ud83d\udfe2 marks until targets attach.',
+ '',
+ ]
+ : []),
...(hasStsoDistribution
? [
- 'The collapsed **STSO distribution** section above buckets every step gap of the sequential-steps run (not a sampled window), split by whether the step ending the gap ran **inline** — in the same warm process as the step before it, so the gap is pure framework overhead — or after a **queue-hop** — the first step of a fresh process, which pays queue dispatch, client reinit and event-log replay. Bars overlay the two runs: `█` is `main`, `┃` marks where this run lands, `░` bridges the gap when this run has more samples in a bucket.',
+ 'The collapsed **STSO distribution** section above buckets every step gap, split **inline** (same warm process — pure framework overhead) vs **queue-hop** (fresh process — dispatch, reinit, replay). `█` = `main`, `┃` = this run, `░` = fill.',
+ '',
+ ]
+ : []),
+ ...(hasCrttDistribution
+ ? [
+ 'The collapsed **CRTT drill-down**: per-variant RTT histograms (fixed log bins, `·` = empty) and mean RTT/positive-CDV profile lines over stream progress and chunk size. Histograms, avgs, and profiles merge exactly across runs; p50–p99 are percentile-of-percentiles. Per-index rows live in the artifacts.',
'',
]
: []),
@@ -666,6 +1019,9 @@ function renderFooter(entries) {
: []),
`Metrics — ${definitions}`,
...(scenarioLegend ? ['', `Scenarios — ${scenarioLegend}`] : []),
+ ...(cadencesLegend
+ ? ['', `Replay cadences (semantic sha256) — ${cadencesLegend}`]
+ : []),
...(targetsLegend
? [
'',
@@ -673,9 +1029,9 @@ function renderFooter(entries) {
]
: []),
'',
- 'All metrics are measured from deployment-side timestamps only. Runs are triggered by an in-deployment route that stamps the anchor (`clientStart`) right before `start()`, so the CI runner’s request and its path through api.vercel.com sit outside every measured window. TTFS = in-deployment `start()` → first step body (turbo uses the in-process fast path, non-turbo the dispatch path), and includes the VQS dispatch hop plus any `/flow` cold start. Fan-out TTFS/TTLS are the first and last step completions of a single `Promise.all` over trivial steps, from the same anchor, so the gap between the two rows is the spread the runtime adds across the fan-out. STSO/WO are measured between step bodies on the deployment. SL is measured inside the workflow (parallel reader/writer steps), so it no longer includes the api.vercel.com read path.',
+ 'All timestamps are deployment-side; runs are triggered in-deployment, so the CI runner and api.vercel.com sit outside every measured window. TTFS = `start()` → first step body (includes dispatch + any cold start); Fan-out TTFS/TTLS = first/last step completion of one `Promise.all` from the same anchor (the gap is the runtime’s fan-out spread); STSO/WO between step bodies; CRTT inside the workflow (excludes the api.vercel.com read path).',
'',
- 'Cold starts are kept in the numbers on purpose — they are part of real bursty-workload latency. The workbench deployment cold-starts the `/flow` invocation for a large fraction of runs, inflating P75+; the **Best** column shows the fastest (warm-start) sample for comparison.',
+ 'Cold starts stay in the numbers (real bursty-workload latency, inflates P75+); **Best** is the warm floor.',
];
// Keep the definitions/methodology out of the way in a collapsed dropdown,
diff --git a/.github/scripts/render-benchmark-comment.test.js b/.github/scripts/render-benchmark-comment.test.js
index b403b51c76..c00a4323ed 100644
--- a/.github/scripts/render-benchmark-comment.test.js
+++ b/.github/scripts/render-benchmark-comment.test.js
@@ -23,6 +23,17 @@ function sampleResult(overrides = {}) {
sequentialIterations: 1,
sequentialStepCount: 1020,
warmupIterations: 2,
+ replayCadences: [
+ {
+ id: 'eve-test-cadence',
+ model: 'test-model',
+ events: 823,
+ spanMs: 6196,
+ totalBytes: 2000000,
+ semanticSha256:
+ '609bc99fb5eb810086dcaecc9128f5fecd7c75d8bc3f2b39a6622f89d5a5a47a',
+ },
+ ],
},
scenarios: [
{ name: 'stream', description: 'one streaming step in turbo mode' },
@@ -120,6 +131,11 @@ test('renders a completed run with a table and embedded history', async () => {
body,
/Scenarios — \*\*stream\*\*: one streaming step in turbo mode/
);
+ // Replay-cadence identity line: full semantic hash on its own legend line
+ assert.match(
+ body,
+ /Replay cadences \(semantic sha256\) — \*\*eve-test-cadence\*\* `609bc99fb5eb810086dcaecc9128f5fecd7c75d8bc3f2b39a6622f89d5a5a47a`<\/sub>/
+ );
// Target marks: TTFS p75 398 > 200 → 🔴; SL row is within target on every
// percentile, so it stays unmarked (no 🟢 anywhere); WO has no targets.
assert.match(body, /398 🔴/);
@@ -580,6 +596,210 @@ function sequentialResult({ inline, queueHop }) {
});
}
+// Fixed log-bin edges matching RTT_HIST_EDGES_MS in the bench helper module
+// (workbench/example/workflows/97_bench_rtt.ts).
+const CRTT_EDGES = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000];
+
+/** Histogram over CRTT_EDGES with counts placed by (value, count) pairs. */
+function crttHist(entries) {
+ const counts = new Array(CRTT_EDGES.length + 1).fill(0);
+ for (const [value, count] of entries) {
+ let bin = 0;
+ while (bin < CRTT_EDGES.length && value >= CRTT_EDGES[bin]) bin++;
+ counts[bin] += count;
+ }
+ return counts;
+}
+
+function crttResult({ avg = 120, hist }) {
+ const streamRow = (scenario, group, extra = {}) => ({
+ metric: 'stream',
+ scenario,
+ unit: 'ms',
+ best: 59,
+ avg,
+ p50: 128,
+ p75: 188,
+ p90: 438,
+ p99: 1229,
+ samples: hist.reduce((a, b) => a + b, 0),
+ raw: [],
+ hist: { edgesMs: CRTT_EDGES, counts: hist },
+ group,
+ bucket: 'all',
+ stream: {
+ iterations: 10,
+ wrCps: 100,
+ wrKiBps: 6.1,
+ rdCps: 99.4,
+ rdKiBps: 6,
+ firstMs: 96,
+ cdvMaxMs: 141,
+ runs: [
+ { wrCps: 100, rdCps: 99.4, firstMs: 96, cdvMaxMs: 141, slipMaxMs: 4 },
+ ],
+ },
+ ...extra,
+ });
+ return sampleResult({
+ scenarios: [
+ { name: 'chunk RTT (llm)', description: 'self-timestamping chunks' },
+ ],
+ metrics: [
+ streamRow('chunk RTT (llm)', 'llm', {
+ progressAvgMs: [110, 112, 115, 113, 118, 120, 119, 125, 130, 135],
+ cdvAvgMs: [2, 2, 3, 5, 9, 15, 24, 40, 66, 108],
+ }),
+ // Artifact-only detail rows: per-index CRTT split and slip tail.
+ {
+ metric: 'crtt',
+ scenario: 'chunk RTT llm (seq 0)',
+ unit: 'ms',
+ best: 97,
+ avg: 130,
+ p50: 112,
+ p75: 126,
+ p90: 129,
+ p99: 157,
+ samples: 10,
+ raw: [],
+ group: 'llm',
+ bucket: 'seq 0',
+ detail: true,
+ },
+ {
+ metric: 'slip',
+ scenario: 'write slip (llm)',
+ unit: 'ms',
+ best: 2,
+ avg: 3,
+ p50: 3,
+ p75: 4,
+ p90: 5,
+ p99: 6,
+ samples: 10,
+ raw: [],
+ group: 'llm',
+ detail: true,
+ },
+ streamRow('replay eve-test (2x)', 'replay', {
+ stream: {
+ iterations: 5,
+ wrCps: 297,
+ wrKiBps: 742,
+ rdCps: 288,
+ rdKiBps: 719,
+ firstMs: 118,
+ cdvMaxMs: 210,
+ runs: [
+ {
+ wrCps: 297,
+ rdCps: 288,
+ firstMs: 118,
+ cdvMaxMs: 210,
+ slipMaxMs: 9,
+ },
+ ],
+ },
+ }),
+ ],
+ });
+}
+
+test('renders stream scenarios in their own table with rate columns', async () => {
+ const { renderComment, extractHistory } = await loadModule();
+ const hist = crttHist([
+ [59, 1400],
+ [128, 1500],
+ [438, 100],
+ ]);
+ const baseline = crttResult({ avg: 150, hist });
+ // Baseline medians differ so deltas render: rd rate was lower on main.
+ baseline.metrics[0].stream.rdCps = 90;
+ const body = renderComment({
+ status: 'completed',
+ results: [crttResult({ avg: 120, hist })],
+ baseline: [baseline],
+ history: [],
+ commit: 'abcdef1234567890',
+ });
+
+ // Stream rows are OUT of the metric table and IN the Streams table.
+ assert.doesNotMatch(body, /\| \*\*stream\*\* \|/);
+ assert.match(
+ body,
+ /\| Scenario \| wr c\/s \| rd c\/s \| wr KiB\/s \| rd KiB\/s \| CRTT 1st \| p75 \| p90 \| p99 \| CDV max \| iters \|/
+ );
+ // Rate cells with plain vs-main deltas, latency cells from percentile
+ // baselines, and NO red/green marks anywhere in the stream table.
+ assert.match(
+ body,
+ /\| chunk RTT \(llm\) \| 100 \(\u00b10%\) \| 99\.4 \(\+10%\) \| 6\.1 \(\u00b10%\) \|/
+ );
+ assert.match(
+ body,
+ /\| replay eve-test \(2x\) \| 297 \(\u00b10%\) \| 288 \(\u00b10%\) \| 742 \(\u00b10%\) \| 719 \(\u00b10%\) \|/
+ );
+ assert.match(body, /\| 141 \(\u00b10%\) \| 10 \|/);
+ const streamsSection = body.slice(
+ body.indexOf('**Streams**'),
+ body.indexOf('')
+ );
+ assert.doesNotMatch(
+ streamsSection,
+ /\ud83d\udd34|\ud83d\udfe2|\ud83d\udd3b|\ud83d\udc9a/
+ );
+ // Detail rows render nowhere.
+ assert.doesNotMatch(body, /seq 0 \|/);
+ assert.doesNotMatch(body, /write slip/);
+ // Drill-down still renders from the stream rows.
+ assert.match(body, /\ud83d\udcc8 CRTT drill-down/);
+ assert.match(
+ body,
+ /llm +\u00b7+[\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588]*\u2588/
+ );
+ assert.match(body, /Delivery jitter over stream progress/);
+ // CRTT + CDV definitions stay in the legend (stream table columns), and
+ // the internal 'stream' id never leaks into it.
+ assert.match(body, /\*\*CRTT\*\*: chunk round-trip time/);
+ assert.match(body, /\*\*CDV\*\*: chunk delay variation/);
+ assert.match(body, /\*\*Streams\*\*: writer\/reader sustained rates/);
+ // History block: per-run arrays and sparkline payloads stripped, medians
+ // and baseline annotations kept.
+ const history = extractHistory(body);
+ const kept = history[0].results[0].metrics[0];
+ assert.strictEqual(kept.hist, undefined);
+ assert.strictEqual(kept.progressAvgMs, undefined);
+ assert.strictEqual(kept.stream.runs, undefined);
+ assert.strictEqual(kept.stream.wrCps, 100);
+ assert.strictEqual(kept.baselineStream.rdCps, 90);
+ // Re-render from history keeps the Streams table, drops the drill-down.
+ const rerendered = renderComment({
+ status: 'running',
+ results: [],
+ history,
+ commit: 'ffffff1234567890',
+ });
+ assert.match(rerendered, /\| chunk RTT \(llm\) \| 100/);
+ assert.doesNotMatch(rerendered, /CRTT drill-down/);
+});
+
+test('renders the stream table without deltas when main has no baseline', async () => {
+ const { renderComment } = await loadModule();
+ const body = renderComment({
+ status: 'completed',
+ results: [crttResult({ hist: crttHist([[128, 3000]]) })],
+ history: [],
+ commit: 'abcdef1234567890',
+ });
+ assert.match(
+ body,
+ /\| chunk RTT \(llm\) \| 100 \| 99\.4 \| 6\.1 \| 6 \| 96 \| 188 \| 438 \| 1229 \| 141 \| 10 \|/
+ );
+ assert.doesNotMatch(body, /%\)/);
+ assert.match(body, /No `main` baseline yet/);
+});
+
test('renders inline and queue-hop STSO histogram diffs against main', async () => {
const { renderComment } = await loadModule();
const body = renderComment({
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 370af08e22..4f5cfe83b2 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -1,8 +1,8 @@
name: Performance Benchmarks
-# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, SL —
-# see packages/core/e2e/benchmark.test.ts for definitions) against a deployed
-# workbench app and posts the results as a sticky PR comment. Re-runs update
+# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, CRTT,
+# CDV — see packages/core/e2e/benchmark.test.ts for definitions) against a
+# deployed workbench app and posts the results as a sticky PR comment. Re-runs update
# the same comment; previous results stay available in a collapsed history
# section (state is embedded in the comment body itself).
#
@@ -96,10 +96,10 @@ jobs:
# backend (e.g. postgres or local), add a matrix entry with
# `world: postgres` / `world: local` and gate the Vercel-specific steps —
# the runner (packages/core/e2e/benchmark.test.ts) already selects its
- # backend from the same env vars as the e2e tests. SL is measured inside the
- # workflow (benchSlWorkflow's parallel reader/writer steps read/write on the
- # deployment), so it no longer needs `run.getReadable()` to work from the
- # test process; the runner only polls returnValue for the collected timings.
+ # backend from the same env vars as the e2e tests. CRTT is measured inside
+ # the workflow (benchCrttWorkflow's parallel reader/writer steps read/write
+ # on the deployment), so it does not need `run.getReadable()` to work from
+ # the test process; the runner only polls returnValue for the aggregates.
benchmark:
name: Benchmark (${{ matrix.target.world }}, ${{ matrix.target.app }})
runs-on: ubuntu-latest
diff --git a/packages/core/e2e/benchmark.test.ts b/packages/core/e2e/benchmark.test.ts
index a64d95e717..5ef838f315 100644
--- a/packages/core/e2e/benchmark.test.ts
+++ b/packages/core/e2e/benchmark.test.ts
@@ -53,24 +53,36 @@
* `(lastStep.end - clientStart) - Σ(step durations)`. Measured on the
* sequential scenario only — on a single-step workflow WO reduces
* algebraically to TTFS.
- * - SL (stream latency): live write->read propagation for the default
- * output stream, measured entirely on the deployment by
- * `benchSlWorkflow`: a reader step and a writer step run in parallel,
- * the reader blocks on the first chunk, and the workflow returns both
- * the writer's `writtenAt` and the reader's `readAt`. SL is
- * `readAt - writtenAt`, so it excludes the api.vercel.com read path
- * the old client-observed metric included.
- * - SO (stream overhead): end-to-end write+consume time in excess of a
- * modelled generation window, measured on the deployment by
- * `benchSoWorkflow`. A writer streams deterministic variable-length
- * LLM-token deltas at a fixed rate for a fixed duration while a
- * parallel reader drains the whole stream; SO is
- * `(doneAt - writtenAt) - chunkCount*intervalMs`, i.e. the
- * overhead/backpressure the stream adds on top of the token rate. Same
- * setup as SL, but the reader stamps `doneAt` after the last chunk
- * rather than `readAt` on the first. Measured for two payload shapes
- * (raw text vs AI-SDK-style structured deltas) so the SO delta between
- * them isolates serialization cost.
+ * - CRTT (chunk round-trip time): per-chunk write->read latency, measured
+ * on the deployment by benchCrttWorkflow. The "round trip" is
+ * deployment -> stream backend -> reader on the SAME deployment
+ * (one clock domain), not an echo to the writer. NAMING: CRTT is
+ * reserved for this same-clock measurement; the future production
+ * cross-clock one-way metric is CTT. Every delta embeds
+ * { seq, writtenAt }; writer and reader run in parallel behind a
+ * reader-ready barrier (chunk 0 is a live delivery). CRTT subsumes
+ * the retired SL/SO rows: SL = the seq-0 slice, SO = last-chunk RTT
+ * + stall accumulation, at ~100x the samples. Aggregation happens
+ * INSIDE the reader step: index buckets (seq 0 / 1-20 / 21+), fixed
+ * log-bin histograms, and mean-RTT profiles over stream progress
+ * and chunk size; the runner merges per-iteration summaries (exact
+ * best/avg/count/hist, percentile-of-percentiles for p50-p99).
+ * Per-index rows are artifact-only (detail: true). No targets yet.
+ * - CDV (chunk delay variation, "delivery jitter"): for seq-adjacent
+ * chunks received back to back, cdv_i = CTT_i - CTT_{i-1}, computed
+ * from RAW unclamped timestamps. Each gap subtracts same-clock
+ * stamps, so CDV is skew-free — the one per-chunk stat measurable
+ * in production across clock domains. Positives are clumps/stalls,
+ * negatives catch-up, means telescope away — the sample unit is
+ * each run's MAX positive cdv. Writer pauses self-exclude, which is
+ * why write slip (writtenAt - scheduledAt vs the open-loop absolute
+ * schedule) stays as artifact-only data: it is the producer-stall
+ * guard neither CRTT nor CDV can see.
+ * - STREAM TABLE: stream scenarios render as one row each in their own
+ * table: writer/reader sustained rates (steady window, 10% trimmed
+ * each side), first-chunk RTT (seq-0, the retired SL signal), CRTT
+ * p75/p90/p99, CDV max. Cells are medians of per-run values (kept
+ * in the artifacts). No 🔴/🟢 marks until targets attach.
*
* Scenarios (defined in workbench/example/workflows/97_bench.ts):
*
@@ -80,18 +92,22 @@
* 4. benchSequentialStepsWorkflow — 1020 trivial sequential steps → STSO + WO
* 5. benchFanOutStepsWorkflow — Promise.all over 100 trivial steps
* → Fan-out TTFS + Fan-out TTLS
- * 6. benchSlWorkflow — parallel reader/writer steps → SL
- * 7. benchSoWorkflow — paced LLM-shaped stream, drained → SO
- * (run in text and structured payload modes)
+ * 6. benchCrttWorkflow — paced stream of self-timestamping chunks →
+ * CRTT/CDV/rates (llm-shaped and size-sweep
+ * variants)
+ * 7. benchReplayWorkflow — replays REAL captured stream cadences
+ * (write instants + chunk sizes from the
+ * capture, speed multiplier the only knob)
+ * → replay rows
*
* Each scenario runs many iterations (env-tunable, see BENCH_* below) so the
* percentiles are computed from real samples.
*
* The backend is selected exactly like the e2e tests (setupWorld): Vercel when
* WORKFLOW_VERCEL_ENV is set, Postgres when WORKFLOW_TARGET_WORLD is
- * @workflow/world-postgres, local filesystem otherwise. Because SL is now
- * measured inside the workflow (not by a reader in this process), it no longer
- * depends on `run.getReadable()` working across processes; CI still runs this
+ * @workflow/world-postgres, local filesystem otherwise. CRTT is measured
+ * inside the workflow (not by a reader in this process), so it does not
+ * depend on `run.getReadable()` working across processes; CI still runs this
* file against Vercel only.
*
* All timestamps are deployment-side, so the only residual skew is intra-Vercel
@@ -99,10 +115,22 @@
* relative to the measured values.
*/
+import { createHash } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { afterAll, beforeAll, describe, test } from 'vitest';
import { getTrustedSourcesHeaders } from '../../../scripts/trusted-sources-headers.mjs';
+import { BENCH_CADENCES } from '../../../workbench/example/workflows/97_bench_cadence';
+import {
+ type BenchDelayTail,
+ type BenchRttMeanProfile,
+ type BenchRttSummary,
+ type BenchSteadyRate,
+ mergeMeanProfiles,
+ mergeRttSummaries,
+ RTT_HIST_EDGES_MS,
+ RTT_INDEX_BUCKETS,
+} from '../../../workbench/example/workflows/97_bench_rtt';
import { getRun } from '../src/runtime';
import { setupWorld } from './utils';
@@ -123,12 +151,13 @@ const envInt = (name: string, fallback: number, min = 1): number => {
return value;
};
-// Iteration counts. The stream/hook/SL scenarios yield one sample per
+// Iteration counts. The stream/hook scenarios yield one sample per
// iteration; the sequential scenario yields (stepCount - 1) STSO samples per
// iteration, so a single long run already provides solid percentiles.
const STREAM_ITERATIONS = envInt('BENCH_STREAM_ITERATIONS', 30);
-const SL_ITERATIONS = envInt('BENCH_SL_ITERATIONS', STREAM_ITERATIONS);
-const SO_ITERATIONS = envInt('BENCH_SO_ITERATIONS', STREAM_ITERATIONS);
+// Each CRTT iteration yields one RTT sample per chunk (300 by default), so
+// few iterations already give thousands of samples per bucket.
+const CRTT_ITERATIONS = envInt('BENCH_CRTT_ITERATIONS', 10);
const SEQUENTIAL_ITERATIONS = envInt('BENCH_SEQUENTIAL_ITERATIONS', 1);
const SEQUENTIAL_STEP_COUNT = envInt('BENCH_SEQUENTIAL_STEP_COUNT', 1020);
// The fan-out scenario yields exactly one TTFS and one TTLS sample per
@@ -151,21 +180,47 @@ const BENCH_METHODOLOGY_VERSION = 2;
// Provisional: now that the proxy leg is out of every window, these will be
// re-tightened once a few in-deployment baselines land.
const TTFS_TARGETS = { p75: 200, p90: 300, p99: 600 };
-const SL_TARGETS = { p75: 50, p90: 60, p99: 125 };
-// SO scenario: model a haiku-size LLM streaming tokens — ~100 tokens/sec, each
-// token a 4-byte chunk, for 3 seconds (300 chunks). The writer paces itself so
-// the write phase spans exactly `SO_CHUNK_COUNT * SO_INTERVAL_MS` ms; SO is the
-// end-to-end write+consume time beyond that window (see runSoIteration). These
-// derive `SO_NOMINAL_DURATION_MS`, the single value subtracted from the
-// measured span, so the workflow's write span and the subtraction never drift.
-const SO_CHUNK_RATE_PER_SEC = envInt('BENCH_SO_CHUNK_RATE', 100);
-const SO_DURATION_SECONDS = envInt('BENCH_SO_DURATION_SECONDS', 3);
-const SO_CHUNK_COUNT = SO_CHUNK_RATE_PER_SEC * SO_DURATION_SECONDS;
-const SO_INTERVAL_MS = 1000 / SO_CHUNK_RATE_PER_SEC;
-const SO_NOMINAL_DURATION_MS = SO_CHUNK_COUNT * SO_INTERVAL_MS;
-// Provisional, like TTFS/SL above: re-tighten once in-deployment baselines land.
-const SO_TARGETS = { p75: 250, p90: 500, p99: 1000 };
+// CRTT workload: model a haiku-size LLM streaming tokens — ~100 tokens/sec
+// for 3 seconds (300 chunks). The writer paces itself so the write phase
+// spans exactly `CRTT_CHUNK_COUNT * CRTT_INTERVAL_MS` ms.
+const CRTT_CHUNK_RATE_PER_SEC = envInt('BENCH_CRTT_CHUNK_RATE', 100);
+const CRTT_DURATION_SECONDS = envInt('BENCH_CRTT_DURATION_SECONDS', 3);
+const CRTT_CHUNK_COUNT = CRTT_CHUNK_RATE_PER_SEC * CRTT_DURATION_SECONDS;
+const CRTT_INTERVAL_MS = 1000 / CRTT_CHUNK_RATE_PER_SEC;
+
+// Replay workload: REAL captured cadences (provenance in
+// 97_bench_cadence.ts) — every write instant and chunk size comes from a
+// capture, one per boundary (eve = demanding envelope protocol, gateway =
+// typical raw SSE). The speed multiplier is the only chosen parameter; 2x
+// matches how real fast-tier models behave (same chunk sizes, compressed
+// time) and exceeds every fast tier measured.
+const REPLAY_SPEED = envInt('BENCH_REPLAY_SPEED', 2);
+const REPLAY_CADENCE_EVE = 'eve-gpt-5.6-sol-2000t'; // 2593 ev / 52.4s / 16.4MiB
+const REPLAY_CADENCE_GATEWAY = 'gateway-gpt-5.4-nano-2000t'; // 1765 ev / 19.9s / 322KiB
+// Eve replays cost ~26s (2x) / ~52s (1x) wall per iteration — few
+// iterations there, more on the cheap gateway row. 1x = reality (not
+// implied by a strained 2x row, and the more linear regression detector);
+// 2x = headroom.
+/**
+ * Cross-system cadence identity: durabench carries its own copy of each
+ * capture, so both sides hash canonical event tuples (format-independent).
+ * CANONICAL FORM (keep in sync with durabench): sha256 over UTF-8
+ * "v1\n" + ",\n" per event, base-10, LF separators.
+ */
+function cadenceSemanticSha256(cadenceId: string): string {
+ const cadence = BENCH_CADENCES[cadenceId];
+ const hash = createHash('sha256');
+ hash.update('v1\n');
+ for (let i = 0; i < cadence.offsetsMs.length; i++) {
+ hash.update(`${cadence.offsetsMs[i]},${cadence.sizes[i]}\n`);
+ }
+ return hash.digest('hex');
+}
+
+const REPLAY_EVE_ITERATIONS = envInt('BENCH_REPLAY_EVE_ITERATIONS', 3);
+const REPLAY_REALITY_ITERATIONS = envInt('BENCH_REPLAY_REALITY_ITERATIONS', 2);
+const REPLAY_GATEWAY_ITERATIONS = envInt('BENCH_REPLAY_GATEWAY_ITERATIONS', 3);
// Guard timeouts so a single stuck run fails fast instead of eating the job.
const RUN_TIMEOUT_MS = envInt('BENCH_RUN_TIMEOUT_MS', 120_000);
@@ -195,17 +250,6 @@ interface BenchStepTiming {
kind: 'inline' | 'queue-hop';
}
-interface BenchStreamLatency {
- writtenAt: number;
- readAt: number;
-}
-
-interface BenchStreamOverhead {
- writtenAt: number;
- doneAt: number;
- received: number;
-}
-
interface StreamIterationResult {
runId: string;
/** `steps[0].start - clientStart`, both deployment-side clocks. */
@@ -240,16 +284,34 @@ interface FanOutIterationResult {
fanOutTtlsMs: number;
}
-interface SlIterationResult {
- runId: string;
- /** `readAt - writtenAt`, both deployment-side step-body clocks. */
- slMs: number;
+/** Mirrors BenchChunkRttResult in workflows/97_bench.ts: per-bucket RTT
+ * summaries aggregated inside the reader step (buckets without samples are
+ * absent). */
+interface BenchChunkRttResult {
+ received: number;
+ all?: BenchRttSummary;
+ byIndex: Partial>;
+ progress?: BenchRttMeanProfile;
+ size?: BenchRttMeanProfile;
+ cdv?: BenchChunkCdv;
+ delivered?: BenchSteadyRate;
}
-interface SoIterationResult {
+/** Mirrors BenchChunkCdv in workflows/97_bench.ts. */
+interface BenchChunkCdv {
+ pairs: number;
+ skippedPairs: number;
+ positive?: BenchDelayTail;
+ progress?: BenchRttMeanProfile;
+}
+
+interface CrttIterationResult {
runId: string;
- /** `(doneAt - writtenAt) - SO_NOMINAL_DURATION_MS`, deployment-side clocks. */
- soMs: number;
+ crtt: BenchChunkRttResult;
+ /** Writer-side pacing slip for the run (artifact-only guard). */
+ writeSlip?: BenchDelayTail;
+ /** Writer-side achieved sustained rate over the steady window. */
+ achieved?: BenchSteadyRate;
}
/** Response shape of the in-deployment `POST /api/bench` trigger route. */
@@ -464,74 +526,133 @@ async function runFanOutIteration(
}
}
-async function runSlIteration(): Promise {
- const { runId } = await triggerBenchRun('benchSlWorkflow');
+async function runCrttIteration(
+ variant: 'llm' | 'sweep',
+ chunkCount: number,
+ intervalMs: number
+): Promise {
+ const { runId } = await triggerBenchRun('benchCrttWorkflow', [
+ chunkCount,
+ intervalMs,
+ variant,
+ ]);
try {
const returnValue = await withTimeout(
getReturnValue(runId),
- RUN_TIMEOUT_MS,
- `benchSlWorkflow returnValue (run ${runId})`
+ // The writer streams for the whole generation window before the run can
+ // complete, so extend the guard past the base run timeout by that window.
+ RUN_TIMEOUT_MS + chunkCount * intervalMs,
+ `benchCrttWorkflow (${variant}) returnValue (run ${runId})`
);
- const sl = (returnValue as { sl?: BenchStreamLatency } | undefined)?.sl;
- if (
- !sl ||
- typeof sl.writtenAt !== 'number' ||
- typeof sl.readAt !== 'number'
- ) {
+ const { crtt, writeSlip, achieved } =
+ (returnValue as
+ | {
+ crtt?: BenchChunkRttResult;
+ writeSlip?: BenchDelayTail;
+ achieved?: BenchSteadyRate;
+ }
+ | undefined) ?? {};
+ if (!crtt || !crtt.all || typeof crtt.all.avg !== 'number') {
throw new Error(
- `Run ${runId} returned no stream-latency sample: ${JSON.stringify(returnValue)?.slice(0, 200)}`
+ `Run ${runId} returned no chunk-RTT summaries: ${JSON.stringify(returnValue)?.slice(0, 200)}`
);
}
- return { runId, slMs: Math.max(0, sl.readAt - sl.writtenAt) };
+ if (crtt.received !== chunkCount) {
+ throw new Error(
+ `Run ${runId} consumed ${crtt.received} chunks, expected ${chunkCount}`
+ );
+ }
+ return { runId, crtt, writeSlip, achieved };
} catch (error) {
(error as Error).message += ` (run ${runId})`;
throw error;
}
}
-async function runSoIteration(
- mode: 'text' | 'structured'
-): Promise {
- const { runId } = await triggerBenchRun('benchSoWorkflow', [
- SO_CHUNK_COUNT,
- SO_INTERVAL_MS,
- mode,
+async function runReplayIteration(
+ cadenceId: string,
+ speed: number
+): Promise {
+ const cadence = BENCH_CADENCES[cadenceId];
+ const { runId } = await triggerBenchRun('benchReplayWorkflow', [
+ cadenceId,
+ speed,
]);
try {
const returnValue = await withTimeout(
getReturnValue(runId),
- // The writer streams for the whole generation window before the run can
- // complete, so extend the guard past the base run timeout by that window.
- RUN_TIMEOUT_MS + SO_NOMINAL_DURATION_MS,
- `benchSoWorkflow (${mode}) returnValue (run ${runId})`
+ RUN_TIMEOUT_MS + cadence.spanMs / speed,
+ `benchReplayWorkflow (${cadenceId} ${speed}x) returnValue (run ${runId})`
);
- const so = (returnValue as { so?: BenchStreamOverhead } | undefined)?.so;
- if (
- !so ||
- typeof so.writtenAt !== 'number' ||
- typeof so.doneAt !== 'number'
- ) {
+ const { crtt, writeSlip, achieved } =
+ (returnValue as
+ | {
+ crtt?: BenchChunkRttResult;
+ writeSlip?: BenchDelayTail;
+ achieved?: BenchSteadyRate;
+ }
+ | undefined) ?? {};
+ if (!crtt || !crtt.all || typeof crtt.all.avg !== 'number') {
throw new Error(
- `Run ${runId} returned no stream-overhead sample: ${JSON.stringify(returnValue)?.slice(0, 200)}`
+ `Run ${runId} returned no chunk-RTT summaries: ${JSON.stringify(returnValue)?.slice(0, 200)}`
);
}
- if (so.received !== SO_CHUNK_COUNT) {
+ if (crtt.received !== cadence.events) {
throw new Error(
- `Run ${runId} consumed ${so.received} chunks, expected ${SO_CHUNK_COUNT}`
+ `Run ${runId} consumed ${crtt.received} chunks, expected ${cadence.events}`
);
}
- // Both timestamps are deployment-side; subtract the modelled generation
- // window and clamp to absorb tiny intra-Vercel skew.
- return {
- runId,
- soMs: Math.max(0, so.doneAt - so.writtenAt - SO_NOMINAL_DURATION_MS),
- };
+ return { runId, crtt, writeSlip, achieved };
} catch (error) {
(error as Error).message += ` (run ${runId})`;
throw error;
}
}
+/**
+ * Median across per-iteration values (undefined skipped); even counts
+ * average the two middles — lower-middle would report the better of a
+ * 2-run scenario's runs and call it the median.
+ */
+function medianOf(values: readonly (number | undefined)[]): number | undefined {
+ const present = values.filter((v): v is number => typeof v === 'number');
+ if (present.length === 0) return undefined;
+ const sorted = [...present].sort((a, b) => a - b);
+ const mid = sorted.length / 2;
+ const median =
+ sorted.length % 2 === 1
+ ? sorted[Math.floor(mid)]
+ : (sorted[mid - 1] + sorted[mid]) / 2;
+ // Inputs carry one decimal; averaging two of them yields float artifacts
+ // (54.650000000000006) that leak into the table and artifacts unrounded.
+ return Math.round(median * 100) / 100;
+}
+
+/**
+ * Records the write-slip detail row for a stream variant (artifact-only,
+ * never rendered). The sample unit is each run's MAX slip: one producer
+ * stall among thousands of chunks vanishes into a pooled p99 but is, by
+ * construction, that run's max. Slip is the guard for producer stalls,
+ * which neither per-chunk RTT (late writes are stamped late) nor CDV
+ * (writer pauses grow both gaps equally) can see.
+ */
+function recordSlipDetailRow(
+ scenario: string,
+ group: string,
+ tails: readonly (BenchDelayTail | undefined)[]
+) {
+ const samples = tails.flatMap((tail) => (tail ? [tail.maxMs] : []));
+ if (samples.length === 0) return;
+ metricRows.push({
+ metric: 'slip',
+ scenario,
+ unit: 'ms',
+ group,
+ detail: true,
+ ...computeStats(samples),
+ });
+}
+
/**
* Runs recorded iterations (plus warmups) sequentially — concurrency would
* contend on the same deployment and skew latencies. Failed iterations are
@@ -600,6 +721,9 @@ interface MetricStats {
best: number;
/** Mean; kept in the JSON for reference but not shown in the PR comment. */
avg: number;
+ /** Median; only recorded for CRTT rows (the exit criteria track median and
+ * average per-chunk RTT). Kept in the JSON, not shown in the PR comment. */
+ p50?: number;
p75: number;
p90: number;
p99: number;
@@ -608,6 +732,56 @@ interface MetricStats {
* comment diffs the whole STSO distribution against `main`, and
* percentiles alone hide *how many* samples moved and by how much. */
raw: number[];
+ /** Fixed-bin histogram of the samples, for rows whose raw samples never
+ * reach this process (CRTT: aggregation happens in the reader step on the
+ * deployment). Fixed shared edges make the PR comment's distribution diff
+ * against `main` exact — the renderer only diffs matching-edge rows. */
+ hist?: { edgesMs: number[]; counts: number[] };
+ /** Drill-down rows (e.g. CRTT per-bucket splits): kept out of the PR
+ * comment's main results table and rendered in a collapsed section. */
+ detail?: boolean;
+ /** Mean RTT per tenth of the stream (CRTT headline rows): the drift/trend
+ * readout, rendered as a progress sparkline in the drill-down. Null
+ * entries are empty bins (rendered as gaps, never as zero). */
+ progressAvgMs?: (number | null)[];
+ /** Mean RTT per log size bin (CRTT sweep headline row): the size→latency
+ * curve, rendered as a size sparkline in the drill-down. Null entries are
+ * bins the sweep left empty. */
+ sizeAvgMs?: (number | null)[];
+ /** Mean POSITIVE CDV per tenth of the stream (stream headline rows),
+ * rendered as a delivery-jitter sparkline in the drill-down: localizes
+ * where delivery clumping/stalls concentrate. Complements the stream
+ * table's CDV max column, which says the worst stall's size but not
+ * where. */
+ cdvAvgMs?: (number | null)[];
+ /** Stream-scenario columns (marks the row for the PR comment's separate
+ * stream table): writer/reader sustained rates over the steady window and
+ * the median worst delivery stall, medians across iterations with the
+ * per-run values retained in `runs`. */
+ stream?: {
+ iterations: number;
+ wrCps?: number;
+ wrKiBps?: number;
+ rdCps?: number;
+ rdKiBps?: number;
+ /** Median across runs of each run's seq-0 RTT — the stream-open path,
+ * before any buffering/backpressure (the retired SL signal). */
+ firstMs?: number;
+ cdvMaxMs?: number;
+ runs: {
+ wrCps?: number;
+ wrKiBps?: number;
+ rdCps?: number;
+ rdKiBps?: number;
+ firstMs?: number;
+ cdvMaxMs?: number;
+ slipMaxMs?: number;
+ }[];
+ };
+ /** Short group/bucket labels for drill-down rendering (CRTT: variant and
+ * index/size bucket). */
+ group?: string;
+ bucket?: string;
}
interface MetricTargets {
@@ -648,6 +822,9 @@ interface MetricRow extends MetricStats {
}
const metricRows: MetricRow[] = [];
+// Per-run seq-0 RTTs from every stream scenario, pooled into the
+// 'first chunk (pooled)' main-table row in afterAll.
+const firstChunkRttSamples: number[] = [];
function recordMetric(
metric: string,
@@ -665,6 +842,131 @@ function recordMetric(
});
}
+/**
+ * Records one CRTT row from per-iteration summaries. Unlike recordMetric
+ * there are no raw samples in this process — the reader step aggregated them
+ * on the deployment — so the row is the mergeRttSummaries merge: exact
+ * count/best/avg/histogram, percentile-of-percentiles for p50-p99. `samples`
+ * is the total chunk count across iterations. The merged fixed-bin histogram
+ * rides along for the PR comment's sparkline drill-down (exact vs `main`,
+ * where the percentiles are approximations). Rows with `detail` are not
+ * rendered at all — they carry the per-index-bucket splits in the results
+ * JSON (with baseline annotations) so a headline regression can be localized
+ * from the artifacts. No targets yet (see the CRTT header note), so no 🔴
+ * marks render.
+ */
+// Mean RTT per profile bin; sums/counts merge exactly across iterations,
+// so these avgs are exact like the histogram. Empty bins become null so
+// the renderer can show them as gaps rather than zeros.
+function profileAvgs(profile?: BenchRttMeanProfile) {
+ return profile?.totalMs.map((total, i) =>
+ profile.counts[i] > 0
+ ? Math.round((total / profile.counts[i]) * 10) / 10
+ : null
+ );
+}
+
+/** Records an artifact-only per-index-bucket CRTT row (never rendered). */
+function recordCrttDetailRow(
+ scenario: string,
+ summaries: readonly (BenchRttSummary | undefined)[],
+ { group, bucket }: { group: string; bucket: string }
+) {
+ const merged = mergeRttSummaries(summaries);
+ if (!merged) return;
+ metricRows.push({
+ metric: 'crtt',
+ scenario,
+ unit: 'ms',
+ best: merged.best,
+ avg: merged.avg,
+ p50: merged.p50,
+ p75: merged.p75,
+ p90: merged.p90,
+ p99: merged.p99,
+ samples: merged.count,
+ raw: [],
+ hist: { edgesMs: RTT_HIST_EDGES_MS, counts: merged.hist },
+ detail: true,
+ group,
+ bucket,
+ });
+}
+
+/**
+ * Records one stream-scenario row (headline of the PR comment's STREAM
+ * table). CRTT percentiles are percentile-of-percentiles across iterations
+ * (count/best/avg/histograms are exact); rates, first-chunk RTT, and CDV
+ * max are medians of per-run values, kept per-run in `stream.runs`.
+ */
+function recordStreamRow(
+ scenario: string,
+ group: string,
+ results: readonly CrttIterationResult[],
+ {
+ size,
+ }: {
+ /** Include the size→latency profile (sweep variant only). */
+ size?: boolean;
+ } = {}
+) {
+ const merged = mergeRttSummaries(results.map((r) => r.crtt.all));
+ if (!merged) return;
+ for (const r of results) {
+ const first = r.crtt.byIndex?.['seq 0']?.avg;
+ if (typeof first === 'number') firstChunkRttSamples.push(first);
+ }
+ const runs = results.map((r) => ({
+ wrCps: r.achieved?.chunksPerSec,
+ wrKiBps: r.achieved?.kibPerSec,
+ rdCps: r.crtt.delivered?.chunksPerSec,
+ rdKiBps: r.crtt.delivered?.kibPerSec,
+ // Single sample (the run's seq-0 chunk), so avg IS that run's value.
+ firstMs: r.crtt.byIndex?.['seq 0']?.avg,
+ cdvMaxMs: r.crtt.cdv?.positive?.maxMs,
+ slipMaxMs: r.writeSlip?.maxMs,
+ }));
+ metricRows.push({
+ metric: 'stream',
+ scenario,
+ unit: 'ms',
+ best: merged.best,
+ avg: merged.avg,
+ p50: merged.p50,
+ p75: merged.p75,
+ p90: merged.p90,
+ p99: merged.p99,
+ samples: merged.count,
+ raw: [],
+ hist: { edgesMs: RTT_HIST_EDGES_MS, counts: merged.hist },
+ group,
+ bucket: 'all',
+ // Nulls (empty bins) are preserved: the renderer draws them as gaps.
+ // Mapping them to 0 would claim "measured no jitter/latency here"
+ // rather than "no samples here" — CDV progress bins are legitimately
+ // empty wherever a tenth of the stream had no positive-cdv chunks.
+ progressAvgMs: profileAvgs(
+ mergeMeanProfiles(results.map((r) => r.crtt.progress))
+ ),
+ sizeAvgMs: size
+ ? profileAvgs(mergeMeanProfiles(results.map((r) => r.crtt.size)))
+ : undefined,
+ cdvAvgMs: profileAvgs(
+ mergeMeanProfiles(results.map((r) => r.crtt.cdv?.progress))
+ ),
+ stream: {
+ iterations: results.length,
+ wrCps: medianOf(runs.map((r) => r.wrCps)),
+ wrKiBps: medianOf(runs.map((r) => r.wrKiBps)),
+ rdCps: medianOf(runs.map((r) => r.rdCps)),
+ rdKiBps: medianOf(runs.map((r) => r.rdKiBps)),
+ firstMs: medianOf(runs.map((r) => r.firstMs)),
+ cdvMaxMs: medianOf(runs.map((r) => r.cdvMaxMs)),
+ runs,
+ },
+ });
+}
+
function getBackend(): string {
if (process.env.WORKFLOW_BENCH_BACKEND) {
return process.env.WORKFLOW_BENCH_BACKEND;
@@ -683,13 +985,22 @@ const SCENARIO_TURBO_STREAM = 'stream';
const SCENARIO_HOOK_STREAM = 'hook + stream';
const SCENARIO_SEQUENTIAL = `${SEQUENTIAL_STEP_COUNT} steps`;
const SCENARIO_FANOUT = `Promise.all(${FANOUT_STEP_COUNT} steps)`;
-const SCENARIO_STREAM_LATENCY = 'stream latency';
-// Two SO scenarios differing only in payload shape. The labels are distinct
-// from the pre-existing 'stream overhead' baseline key, so the payload change
-// doesn't diff against the old fixed-'aaaa' numbers — the SO deltas start blank
-// and re-baseline on the next `main` run.
-const SCENARIO_STREAM_OVERHEAD_TEXT = 'stream overhead (text)';
-const SCENARIO_STREAM_OVERHEAD_STRUCTURED = 'stream overhead (structured)';
+// Stream scenario labels, doubling as the stream-table rows' scenario keys;
+// the per-bucket detail rows are keyed `paced control ()` and slip
+// detail rows `write slip ()`. All new baseline keys, so the
+// stream deltas stay blank until `main` produces them.
+// Synthetic rows are named for their role: the metronome is the control
+// (diagnostic anchor + flush-cadence probe); the sweep isolates size
+// causally (rotation decouples size from position; the replay ramp
+// couples them).
+const SCENARIO_PACED_CONTROL = 'paced control (100/s, 60B)';
+const SCENARIO_SIZE_SWEEP = 'size sweep (100/s, 160B-12KB)';
+// Capture id + speed IS the baseline key: a re-capture is a new workload
+// and starts a new baseline by construction.
+// Parenthesized speed, not `@2x`: GitHub renders @ as a user mention.
+const SCENARIO_REPLAY_EVE = `replay ${REPLAY_CADENCE_EVE} (${REPLAY_SPEED}x)`;
+const SCENARIO_REPLAY_REALITY = `replay ${REPLAY_CADENCE_EVE} (1x)`;
+const SCENARIO_REPLAY_GATEWAY = `replay ${REPLAY_CADENCE_GATEWAY} (1x)`;
const SCENARIO_DESCRIPTIONS = [
{
name: SCENARIO_STEP,
@@ -715,19 +1026,34 @@ const SCENARIO_DESCRIPTIONS = [
description: `${FANOUT_STEP_COUNT} trivial no-op steps started together in a single Promise.all; Fan-out TTFS is the first of them to complete and Fan-out TTLS the last, both from the in-deployment clientStart, so their gap is the spread the runtime adds across the fan-out`,
},
{
- name: SCENARIO_STREAM_LATENCY,
- description:
- 'parallel reader/writer steps on a dedicated stream; SL is the in-deployment write->read propagation (readAt - writtenAt)',
+ name: SCENARIO_PACED_CONTROL,
+ description: `the control: ${CRTT_CHUNK_COUNT} tiny (~60B) deltas metronome-paced at ${CRTT_CHUNK_RATE_PER_SEC}/s — zero workload structure, so it reads the transport floor and flush cadence, and disambiguates transport-wide vs workload-specific when a replay row moves`,
+ },
+ {
+ name: SCENARIO_SIZE_SWEEP,
+ description: `same pacing as the control with deltas padded in rotation across seven log-spaced sizes (~160B–12KB) — rotation decouples size from stream position, so it isolates whether chunk size causes latency`,
+ },
+ {
+ name: SCENARIO_REPLAY_GATEWAY,
+ description: `raw provider SSE cadence captured at the AI gateway boundary (gpt-5.4-nano, the most popular gateway model; per-token deltas p50 208B = the modal production chunk size), replayed exactly as measured — the typical customer's workload; its CDV is the typical customer's real delivery jitter`,
+ },
+ {
+ name: SCENARIO_REPLAY_REALITY,
+ description: `a captured eve turn (gpt-5.6-sol, the most-used demanding eve model; ~2000 output tokens = production p50 turn length) replayed exactly as measured — eve's envelope protocol re-ships the cumulative message so sizes ramp 142B→13KB; the demanding outlier tenant's reality`,
},
{
- name: SCENARIO_STREAM_OVERHEAD_TEXT,
- description: `writer streams ${SO_CHUNK_COUNT} variable-length text token deltas paced at ${SO_CHUNK_RATE_PER_SEC}/s for ${SO_DURATION_SECONDS}s (a haiku-size LLM's token throughput) while a parallel reader drains the whole stream; SO is the end-to-end write+consume time beyond the ${SO_DURATION_SECONDS}s generation window (overhead/backpressure)`,
+ name: SCENARIO_REPLAY_EVE,
+ description: `the same eve capture at ${REPLAY_SPEED}x — the headroom/stress row; real fast-tier models emit the same chunk sizes at proportionally higher rate, so time compression is a faithful speed model`,
},
{
- name: SCENARIO_STREAM_OVERHEAD_STRUCTURED,
- description: `same workload as ${SCENARIO_STREAM_OVERHEAD_TEXT}, but each delta is an AI-SDK-style structured object ({ type: 'text-delta', id, text }) instead of a raw string, so the SO gap vs the text scenario is the added serialization cost`,
+ name: 'first chunk (pooled)',
+ description: `every run's seq-0 RTT pooled across all stream scenarios — the first chunk precedes any workload differentiation, so pooling samples one shared stream-open path with exact percentiles`,
},
];
+// Cross-system cadence identity: the full semantic hash lands in
+// config.replayCadences (rendered as its own "Replay cadences" legend line
+// and copyable from the artifacts); durabench computes the same hash over
+// its copy (see cadenceSemanticSha256).
// Datadog APM permalink for a trace id. The benchmark deployment exports its
// OTel spans to Datadog, and `/api/bench` returns the trace id of the request
@@ -827,52 +1153,114 @@ describe('workflow benchmarks', () => {
}
);
- test('scenario: stream latency', { timeout: 30 * 60_000 }, async () => {
+ test('scenario: paced control', { timeout: 30 * 60_000 }, async () => {
const results = await runScenario(
- SCENARIO_STREAM_LATENCY,
- SL_ITERATIONS,
- () => runSlIteration()
+ SCENARIO_PACED_CONTROL,
+ CRTT_ITERATIONS,
+ () => runCrttIteration('llm', CRTT_CHUNK_COUNT, CRTT_INTERVAL_MS)
);
- recordMetric(
- 'sl',
- SCENARIO_STREAM_LATENCY,
- results.map((r) => r.slMs),
- SL_TARGETS
+ // One rendered row per stream scenario (in the separate stream table);
+ // the index-bucket rows split RTT by position in the stream for the
+ // results artifacts only (flat across runs so far), as do slip tails.
+ recordStreamRow(SCENARIO_PACED_CONTROL, 'control', results);
+ for (const bucket of RTT_INDEX_BUCKETS) {
+ recordCrttDetailRow(
+ `paced control (${bucket})`,
+ results.map((r) => r.crtt.byIndex[bucket]),
+ { group: 'control', bucket }
+ );
+ }
+ recordSlipDetailRow(
+ 'write slip (paced control)',
+ 'control',
+ results.map((r) => r.writeSlip)
);
});
+ test('scenario: size sweep', { timeout: 30 * 60_000 }, async () => {
+ const results = await runScenario(
+ SCENARIO_SIZE_SWEEP,
+ CRTT_ITERATIONS,
+ () => runCrttIteration('sweep', CRTT_CHUNK_COUNT, CRTT_INTERVAL_MS)
+ );
+ // The size→latency profile only makes sense here: the llm-shaped
+ // deltas all land in the smallest size bin.
+ recordStreamRow(SCENARIO_SIZE_SWEEP, 'sweep', results, {
+ size: true,
+ });
+ recordSlipDetailRow(
+ 'write slip (size sweep)',
+ 'sweep',
+ results.map((r) => r.writeSlip)
+ );
+ });
+
+ // The replay scenarios run (and therefore render) in ascending difficulty:
+ // gateway 1x (typical customer as measured, the lightest total load) →
+ // eve 1x (demanding workload as measured) → eve 2x (stress). The gateway
+ // capture deliberately has no 2x row: nano at 1x already sits at the fast
+ // end of measured gateway rates, and a first run showed gateway-2x tails
+ // statistically identical to eve 2x — headroom is eve 2x's job.
+
test(
- 'scenario: stream overhead (text)',
+ 'scenario: replay (gateway gpt-5.4-nano 2000t, 1x reality)',
{ timeout: 30 * 60_000 },
async () => {
const results = await runScenario(
- SCENARIO_STREAM_OVERHEAD_TEXT,
- SO_ITERATIONS,
- () => runSoIteration('text')
+ SCENARIO_REPLAY_GATEWAY,
+ REPLAY_GATEWAY_ITERATIONS,
+ () => runReplayIteration(REPLAY_CADENCE_GATEWAY, 1),
+ // ~20s wall per run; one warmup — earlier scenarios already warmed
+ // the deployment and stream path.
+ { warmupIterations: 1 }
);
- recordMetric(
- 'so',
- SCENARIO_STREAM_OVERHEAD_TEXT,
- results.map((r) => r.soMs),
- SO_TARGETS
+ recordStreamRow(SCENARIO_REPLAY_GATEWAY, 'gw 1x', results);
+ recordSlipDetailRow(
+ `write slip (${REPLAY_CADENCE_GATEWAY} 1x)`,
+ 'gw 1x',
+ results.map((r) => r.writeSlip)
);
}
);
test(
- 'scenario: stream overhead (structured)',
+ 'scenario: replay (eve gpt-5.6-sol 2000t, 1x reality)',
{ timeout: 30 * 60_000 },
async () => {
const results = await runScenario(
- SCENARIO_STREAM_OVERHEAD_STRUCTURED,
- SO_ITERATIONS,
- () => runSoIteration('structured')
+ SCENARIO_REPLAY_REALITY,
+ REPLAY_REALITY_ITERATIONS,
+ () => runReplayIteration(REPLAY_CADENCE_EVE, 1),
+ // ~52s wall per run; no warmup — the gateway 1x replay just ran, so
+ // the replay path is warm.
+ { warmupIterations: 0 }
);
- recordMetric(
- 'so',
- SCENARIO_STREAM_OVERHEAD_STRUCTURED,
- results.map((r) => r.soMs),
- SO_TARGETS
+ recordStreamRow(SCENARIO_REPLAY_REALITY, 'eve 1x', results);
+ recordSlipDetailRow(
+ `write slip (${REPLAY_CADENCE_EVE} 1x)`,
+ 'eve 1x',
+ results.map((r) => r.writeSlip)
+ );
+ }
+ );
+
+ test(
+ 'scenario: replay (eve gpt-5.6-sol 2000t, 2x)',
+ { timeout: 30 * 60_000 },
+ async () => {
+ const results = await runScenario(
+ SCENARIO_REPLAY_EVE,
+ REPLAY_EVE_ITERATIONS,
+ () => runReplayIteration(REPLAY_CADENCE_EVE, REPLAY_SPEED),
+ // No warmup: the same cadence already replayed at 1x above, so
+ // everything this touches is warm.
+ { warmupIterations: 0 }
+ );
+ recordStreamRow(SCENARIO_REPLAY_EVE, 'eve 2x', results);
+ recordSlipDetailRow(
+ `write slip (${REPLAY_CADENCE_EVE} ${REPLAY_SPEED}x)`,
+ 'eve 2x',
+ results.map((r) => r.writeSlip)
);
}
);
@@ -970,6 +1358,18 @@ describe('workflow benchmarks', () => {
});
afterAll(() => {
+ // Pooled first-chunk RTTs: seq 0 precedes any workload differentiation
+ // (no queue depth / backpressure), so every scenario samples one shared
+ // stream-open path — the one valid cross-scenario pool. ~TTFS-sized
+ // sample set, exact percentiles (raw samples, no merge).
+ if (firstChunkRttSamples.length > 0) {
+ metricRows.push({
+ metric: 'crtt',
+ scenario: 'first chunk (pooled)',
+ unit: 'ms',
+ ...computeStats(firstChunkRttSamples),
+ });
+ }
if (metricRows.length === 0) {
console.warn('[bench] No metrics collected; skipping results file');
return;
@@ -992,11 +1392,29 @@ describe('workflow benchmarks', () => {
commit: process.env.GITHUB_SHA || undefined,
config: {
streamIterations: STREAM_ITERATIONS,
- slIterations: SL_ITERATIONS,
- soIterations: SO_ITERATIONS,
- soChunkCount: SO_CHUNK_COUNT,
- soChunkRatePerSec: SO_CHUNK_RATE_PER_SEC,
- soDurationSeconds: SO_DURATION_SECONDS,
+ crttIterations: CRTT_ITERATIONS,
+ crttChunkCount: CRTT_CHUNK_COUNT,
+ crttChunkRatePerSec: CRTT_CHUNK_RATE_PER_SEC,
+ crttDurationSeconds: CRTT_DURATION_SECONDS,
+ replaySpeed: REPLAY_SPEED,
+ replayEveIterations: REPLAY_EVE_ITERATIONS,
+ replayRealityIterations: REPLAY_REALITY_ITERATIONS,
+ replayGatewayIterations: REPLAY_GATEWAY_ITERATIONS,
+ replayCadences: [REPLAY_CADENCE_EVE, REPLAY_CADENCE_GATEWAY].map(
+ (id) => {
+ const c = BENCH_CADENCES[id];
+ return {
+ id,
+ model: c.model,
+ capturedAt: c.capturedAt,
+ eveCommit: c.eveCommit,
+ events: c.events,
+ spanMs: c.spanMs,
+ totalBytes: c.totalBytes,
+ semanticSha256: cadenceSemanticSha256(id),
+ };
+ }
+ ),
sequentialIterations: SEQUENTIAL_ITERATIONS,
sequentialStepCount: SEQUENTIAL_STEP_COUNT,
fanoutIterations: FANOUT_ITERATIONS,
diff --git a/packages/core/src/bench-chunk-rtt-stats.test.ts b/packages/core/src/bench-chunk-rtt-stats.test.ts
new file mode 100644
index 0000000000..6a40509db0
--- /dev/null
+++ b/packages/core/src/bench-chunk-rtt-stats.test.ts
@@ -0,0 +1,377 @@
+/**
+ * Unit tests for the chunk-RTT (CRTT) benchmark's pure bucketing/aggregation
+ * helpers (workbench/example/workflows/97_bench_rtt.ts). The module is
+ * dependency-free on purpose: the same code runs inside the benchmark's
+ * reader step on the deployment (per-iteration aggregation) and in the
+ * benchmark runner (cross-iteration merging), and this suite is the fast
+ * check on both — the bench itself only runs against a deployment.
+ */
+
+import { describe, expect, test } from 'vitest';
+import {
+ type BenchRttSummary,
+ type CdvArrival,
+ computeCdv,
+ histogramRttSamples,
+ mergeMeanProfiles,
+ mergeRttSummaries,
+ progressProfile,
+ RTT_HIST_EDGES_MS,
+ RTT_INDEX_BUCKETS,
+ RTT_PROGRESS_BINS,
+ RTT_SIZE_BIN_EDGES_BYTES,
+ rttIndexBucket,
+ rttSizeBin,
+ sizeProfile,
+ steadyRate,
+ summarizeDelayTail,
+ summarizeRttSamples,
+} from '../../../workbench/example/workflows/97_bench_rtt';
+
+/** Histogram with `count` in the bin holding `value` and zeros elsewhere. */
+function histWith(value: number, count = 1): number[] {
+ const hist = new Array(RTT_HIST_EDGES_MS.length + 1).fill(0);
+ let bin = 0;
+ while (bin < RTT_HIST_EDGES_MS.length && value >= RTT_HIST_EDGES_MS[bin]) {
+ bin++;
+ }
+ hist[bin] = count;
+ return hist;
+}
+
+describe('rttIndexBucket', () => {
+ test('boundaries: stream-open write / warmup / steady state', () => {
+ expect(rttIndexBucket(0)).toBe('seq 0');
+ expect(rttIndexBucket(1)).toBe('seq 1-20');
+ expect(rttIndexBucket(20)).toBe('seq 1-20');
+ expect(rttIndexBucket(21)).toBe('seq 21+');
+ expect(rttIndexBucket(299)).toBe('seq 21+');
+ });
+
+ test('every bucket is a declared bucket key', () => {
+ for (let seq = 0; seq < 300; seq++) {
+ expect(RTT_INDEX_BUCKETS).toContain(rttIndexBucket(seq));
+ }
+ });
+});
+
+describe('progressProfile', () => {
+ test('bins by fraction of the stream, so profiles are chunk-count independent', () => {
+ // 300 chunks: each tenth holds exactly 30.
+ const rtts = Array.from({ length: 300 }, (_, seq) => seq);
+ const profile = progressProfile(rtts);
+ expect(profile.counts).toEqual(new Array(RTT_PROGRESS_BINS).fill(30));
+ // First tenth: seq 0..29 (sum 435); last tenth: seq 270..299 (sum 8535).
+ expect(profile.totalMs[0]).toBe(435);
+ expect(profile.totalMs[RTT_PROGRESS_BINS - 1]).toBe(8535);
+
+ // 20 chunks (fewer than would fill 10 bins evenly at other counts): still
+ // 2 per tenth.
+ const small = progressProfile(Array.from({ length: 20 }, () => 5));
+ expect(small.counts).toEqual(new Array(RTT_PROGRESS_BINS).fill(2));
+ });
+
+ test('skips sparse entries defensively', () => {
+ const rtts: (number | undefined)[] = new Array(100);
+ rtts[0] = 7;
+ rtts[99] = 9;
+ const profile = progressProfile(rtts);
+ expect(profile.counts.reduce((a, b) => a + b, 0)).toBe(2);
+ expect(profile.totalMs[0]).toBe(7);
+ expect(profile.totalMs[RTT_PROGRESS_BINS - 1]).toBe(9);
+ });
+});
+
+describe('mergeMeanProfiles', () => {
+ test('returns undefined with no profiles and sums exactly otherwise', () => {
+ expect(mergeMeanProfiles([])).toBeUndefined();
+ expect(mergeMeanProfiles([undefined])).toBeUndefined();
+ const a = progressProfile(Array.from({ length: 10 }, () => 10));
+ const b = progressProfile(Array.from({ length: 10 }, () => 30));
+ const merged = mergeMeanProfiles([a, undefined, b]);
+ expect(merged?.counts).toEqual(new Array(RTT_PROGRESS_BINS).fill(2));
+ expect(merged?.totalMs).toEqual(new Array(RTT_PROGRESS_BINS).fill(40));
+ });
+});
+
+describe('sizeProfile', () => {
+ test('bins by serialized size with doubling edges', () => {
+ expect(rttSizeBin(100)).toBe(0);
+ expect(rttSizeBin(255)).toBe(0);
+ // A size exactly on an edge lands in the bin the edge opens.
+ expect(rttSizeBin(256)).toBe(1);
+ expect(rttSizeBin(1024)).toBe(3);
+ expect(rttSizeBin(8192)).toBe(RTT_SIZE_BIN_EDGES_BYTES.length);
+ expect(rttSizeBin(20000)).toBe(RTT_SIZE_BIN_EDGES_BYTES.length);
+ });
+
+ test('the sweep pad ladder occupies every size bin exactly once', () => {
+ // Approximate serialized sizes of the sweep rotation: ~60B base chunk
+ // plus pads of 100/340/700/1400/3000/6000/12000 chars.
+ const sizes = [160, 400, 760, 1460, 3060, 6060, 12060];
+ expect(new Set(sizes.map(rttSizeBin)).size).toBe(
+ RTT_SIZE_BIN_EDGES_BYTES.length + 1
+ );
+ });
+
+ test('accumulates count and total RTT per bin', () => {
+ const profile = sizeProfile([
+ { bytes: 160, rttMs: 10 },
+ { bytes: 200, rttMs: 20 },
+ { bytes: 12060, rttMs: 50 },
+ ]);
+ expect(profile.counts[0]).toBe(2);
+ expect(profile.totalMs[0]).toBe(30);
+ expect(profile.counts[RTT_SIZE_BIN_EDGES_BYTES.length]).toBe(1);
+ expect(profile.totalMs[RTT_SIZE_BIN_EDGES_BYTES.length]).toBe(50);
+ expect(profile.counts.reduce((a, b) => a + b, 0)).toBe(3);
+ });
+});
+
+describe('histogramRttSamples', () => {
+ test('bins are [prev edge, edge), first bin is <1ms, last is 5000+', () => {
+ expect(histogramRttSamples([0, 0.5])[0]).toBe(2);
+ // A sample exactly on an edge lands in the bin the edge opens.
+ const atEdge = histogramRttSamples([1]);
+ expect(atEdge[0]).toBe(0);
+ expect(atEdge[1]).toBe(1);
+ const overflow = histogramRttSamples([5000, 60000]);
+ expect(overflow[RTT_HIST_EDGES_MS.length]).toBe(2);
+ });
+
+ test('counts sum to the sample count', () => {
+ const samples = [0, 1, 3, 7, 59, 128, 438, 1229, 9999];
+ const hist = histogramRttSamples(samples);
+ expect(hist).toHaveLength(RTT_HIST_EDGES_MS.length + 1);
+ expect(hist.reduce((a, b) => a + b, 0)).toBe(samples.length);
+ });
+});
+
+describe('summarizeRttSamples', () => {
+ test('returns undefined for an empty bucket', () => {
+ expect(summarizeRttSamples([])).toBeUndefined();
+ });
+
+ test('single sample collapses every stat to that value', () => {
+ expect(summarizeRttSamples([7])).toEqual({
+ count: 1,
+ best: 7,
+ avg: 7,
+ hist: histWith(7),
+ p50: 7,
+ p75: 7,
+ p90: 7,
+ p99: 7,
+ });
+ });
+
+ test('percentiles use the runner convention (nearest-rank via ceil)', () => {
+ // 1..100 shuffled: pQ must be exactly Q under nearest-rank.
+ const samples = Array.from({ length: 100 }, (_, i) => i + 1).sort(
+ () => 0.5 - Math.random()
+ );
+ expect(summarizeRttSamples(samples)).toEqual({
+ count: 100,
+ best: 1,
+ avg: 50.5,
+ hist: histogramRttSamples(samples),
+ p50: 50,
+ p75: 75,
+ p90: 90,
+ p99: 99,
+ });
+ });
+
+ test('rounds to 0.1ms', () => {
+ const summary = summarizeRttSamples([1, 2, 2.44]);
+ expect(summary?.avg).toBe(1.8);
+ expect(summary?.p99).toBe(2.4);
+ });
+});
+
+describe('summarizeDelayTail', () => {
+ test('returns undefined for no samples', () => {
+ expect(summarizeDelayTail([])).toBeUndefined();
+ });
+
+ test('max catches a single stall that pooled percentiles would hide', () => {
+ // 299 jitter-floor samples plus ONE 800ms stall.
+ const samples = [...Array.from({ length: 299 }, () => 2), 800];
+ const tail = summarizeDelayTail(samples);
+ expect(tail?.maxMs).toBe(800);
+ // Even p99 over the pooled run misses a 1-in-300 stall (nearest-rank
+ // p99 of 300 samples is the 297th) — which is why the runner reports
+ // per-run max, not pooled percentiles.
+ expect(tail?.p99Ms).toBe(2);
+ expect(tail?.count).toBe(300);
+ expect(tail?.avgMs).toBe(4.7);
+ });
+});
+
+describe('steadyRate', () => {
+ test('returns undefined when the window cannot define a rate', () => {
+ expect(steadyRate([])).toBeUndefined();
+ expect(steadyRate([{ atMs: 0, bytes: 100 }])).toBeUndefined();
+ // Same-instant points: zero span.
+ expect(
+ steadyRate([
+ { atMs: 5, bytes: 1 },
+ { atMs: 5, bytes: 1 },
+ ])
+ ).toBeUndefined();
+ });
+
+ test('computes chunks/s and KiB/s over the trimmed steady window', () => {
+ // 100 chunks of 1024B at exactly 10ms spacing → 100 c/s, 100 KiB/s.
+ const points = Array.from({ length: 100 }, (_, i) => ({
+ atMs: i * 10,
+ bytes: 1024,
+ }));
+ const rate = steadyRate(points);
+ expect(rate?.windowChunks).toBe(80); // 10% trimmed each side
+ expect(rate?.chunksPerSec).toBe(100);
+ // Bytes are counted over the window's 79 intervals (the first point's
+ // bytes predate the window's clock): 79 KiB over 790ms = exactly the
+ // stream's true steady rate, with no 1/(n-1) inflation.
+ expect(rate?.kibPerSec).toBe(100);
+ });
+
+ test('trimming excludes warmup and drain from the sustained rate', () => {
+ // A slow first and last chunk (cold start / final flush) that would
+ // wreck the naive whole-run rate.
+ const points = [
+ { atMs: 0, bytes: 100 },
+ ...Array.from({ length: 20 }, (_, i) => ({
+ atMs: 1000 + i * 10,
+ bytes: 100,
+ })),
+ { atMs: 10_000, bytes: 100 },
+ ];
+ const rate = steadyRate(points);
+ // Steady window covers only the 10ms-spaced middle → ~100 c/s, not the
+ // ~2 c/s the whole-run span would suggest.
+ expect(rate?.chunksPerSec).toBeGreaterThan(90);
+ });
+});
+
+describe('computeCdv', () => {
+ // Chunks written every 10ms, delivered in clumps of three: the first of
+ // each clump waits for the flush, the other two arrive ~together.
+ const clumped = (): CdvArrival[] => [
+ { seq: 0, writtenAt: 1000, readAt: 1030 },
+ { seq: 1, writtenAt: 1010, readAt: 1030 },
+ { seq: 2, writtenAt: 1020, readAt: 1031 },
+ { seq: 3, writtenAt: 1030, readAt: 1060 },
+ { seq: 4, writtenAt: 1040, readAt: 1060 },
+ { seq: 5, writtenAt: 1050, readAt: 1061 },
+ ];
+
+ test('clumped delivery reads as negative catch-up plus positive stalls', () => {
+ const { cdvMs, skippedPairs } = computeCdv(clumped());
+ // (readGap - writeGap) per pair: (0-10), (1-10), (29-10), (0-10), (1-10).
+ expect(cdvMs).toEqual([-10, -9, 19, -10, -9]);
+ expect(skippedPairs).toBe(0);
+ });
+
+ test('equals the telescoping identity cdv_i = CTT_i - CTT_{i-1}', () => {
+ const arrivals = clumped();
+ const ctt = arrivals.map((a) => a.readAt - a.writtenAt);
+ const { cdvMs } = computeCdv(arrivals);
+ expect(cdvMs).toEqual(ctt.slice(1).map((v, i) => v - ctt[i]));
+ // ...and the signed sum telescopes to CTT_last - CTT_first.
+ expect(cdvMs.reduce((a, b) => a + b, 0)).toBe(ctt[ctt.length - 1] - ctt[0]);
+ });
+
+ test('is immune to a constant clock offset between writer and reader', () => {
+ const skewed = clumped().map((a) => ({ ...a, readAt: a.readAt - 5000 }));
+ // Reader clock 5s behind the writer: every CTT is negative, CDV is
+ // untouched — each gap subtracts same-clock stamps.
+ expect(computeCdv(skewed).cdvMs).toEqual(computeCdv(clumped()).cdvMs);
+ });
+
+ test('positive cdv is indexed by the later seq, padded to the stream', () => {
+ const { positiveBySeq } = computeCdv(clumped());
+ expect(positiveBySeq.length).toBe(6);
+ expect(positiveBySeq[3]).toBe(19);
+ expect(positiveBySeq.filter((v) => v !== undefined)).toEqual([19]);
+ });
+
+ test('counts duplicates, reorders, and non-adjacent pairs', () => {
+ const arrivals: CdvArrival[] = [
+ { seq: 0, writtenAt: 1000, readAt: 1030 },
+ { seq: 2, writtenAt: 1020, readAt: 1050 }, // hole: skipped pair
+ { seq: 1, writtenAt: 1010, readAt: 1051 }, // reorder: skipped pair
+ { seq: 1, writtenAt: 1010, readAt: 1052 }, // duplicate + not adjacent
+ ];
+ const cdv = computeCdv(arrivals);
+ expect(cdv.cdvMs).toEqual([]);
+ expect(cdv.duplicateSeqs).toBe(1);
+ expect(cdv.reorderedArrivals).toBe(1);
+ expect(cdv.skippedPairs).toBe(3);
+ });
+
+ test('a single chunk has no pair', () => {
+ const cdv = computeCdv([{ seq: 0, writtenAt: 1000, readAt: 1030 }]);
+ expect(cdv.cdvMs).toEqual([]);
+ expect(cdv.skippedPairs).toBe(0);
+ });
+});
+
+describe('mergeRttSummaries', () => {
+ const summary = (overrides: Partial): BenchRttSummary => ({
+ count: 10,
+ best: 1,
+ avg: 5,
+ hist: histWith(5, 10),
+ p50: 5,
+ p75: 6,
+ p90: 8,
+ p99: 9,
+ ...overrides,
+ });
+
+ test('returns undefined when no iteration produced the bucket', () => {
+ expect(mergeRttSummaries([])).toBeUndefined();
+ expect(mergeRttSummaries([undefined, undefined])).toBeUndefined();
+ });
+
+ test('single summary passes through unchanged', () => {
+ const s = summary({});
+ expect(mergeRttSummaries([undefined, s])).toEqual(s);
+ });
+
+ test('count sums, best is the min, avg is count-weighted', () => {
+ const merged = mergeRttSummaries([
+ summary({ count: 10, best: 2, avg: 10 }),
+ summary({ count: 30, best: 1, avg: 2 }),
+ ]);
+ expect(merged?.count).toBe(40);
+ expect(merged?.best).toBe(1);
+ expect(merged?.avg).toBe(4); // (10*10 + 2*30) / 40
+ });
+
+ test('histograms merge by elementwise summation (exact)', () => {
+ const merged = mergeRttSummaries([
+ summary({ count: 10, hist: histWith(5, 10) }),
+ summary({ count: 30, hist: histWith(128, 30) }),
+ ]);
+ const expected = histWith(5, 10);
+ const bin128 = histWith(128, 30);
+ for (let i = 0; i < expected.length; i++) expected[i] += bin128[i];
+ expect(merged?.hist).toEqual(expected);
+ expect(merged?.hist.reduce((a, b) => a + b, 0)).toBe(40);
+ });
+
+ test('percentiles merge as percentile-of-percentiles', () => {
+ const summaries = Array.from({ length: 10 }, (_, i) =>
+ summary({ p50: i + 1, p90: (i + 1) * 10, p99: (i + 1) * 100 })
+ );
+ const merged = mergeRttSummaries(summaries);
+ // p50 over the ten per-iteration p50s (1..10) = 5.
+ expect(merged?.p50).toBe(5);
+ // p90 over 10..100 = 90.
+ expect(merged?.p90).toBe(90);
+ // p99 over 100..1000 = max of maxes (exact at the tail).
+ expect(merged?.p99).toBe(1000);
+ });
+});
diff --git a/workbench/example/workflows/97_bench.ts b/workbench/example/workflows/97_bench.ts
index a940ee33a8..3bb1e5edaf 100644
--- a/workbench/example/workflows/97_bench.ts
+++ b/workbench/example/workflows/97_bench.ts
@@ -27,9 +27,40 @@
// overhead/backpressure. Two payload shapes are supported so the runner can
// isolate serialization cost: `'text'` (raw string fragments) and
// `'structured'` (AI-SDK-style `{ type: 'text-delta', id, text }` objects).
+// - `benchCrttWorkflow` measures per-chunk round-trip time (CRTT), reusing the
+// SO setup (paced writer + parallel reader, same deployment, so no clock
+// skew beyond intra-Vercel NTP bounds) but embedding `{ seq, writtenAt }` in
+// every chunk — the SL scenario's payload-embedded-timestamp trick applied
+// to the whole stream. The "round trip" is deployment -> stream backend ->
+// reader on the same deployment (one clock domain), not an echo back to the
+// writer. The reader stamps each chunk's arrival, computes
+// `rtt = Date.now() - chunk.writtenAt`, and aggregates on the deployment
+// (see 97_bench_rtt.ts): chunk-index buckets, mean-RTT profiles over stream
+// progress and over serialized chunk size, and fixed log-bin histograms —
+// compact aggregates instead of hundreds of raw samples.
+// - `benchReplayWorkflow` reuses the whole CRTT measurement rig but replays a
+// REAL captured stream cadence (97_bench_cadence.ts): every write
+// instant and chunk size comes from the capture, so the replay scenario's
+// only chosen parameter is the speed multiplier.
import { createHook, getWorkflowMetadata, getWritable } from 'workflow';
import { getRun } from 'workflow/api';
+import { BENCH_CADENCES } from './97_bench_cadence';
+import {
+ type BenchDelayTail,
+ type BenchRttMeanProfile,
+ type BenchRttSummary,
+ type BenchSteadyRate,
+ type CdvArrival,
+ computeCdv,
+ progressProfile,
+ type RttIndexBucket,
+ rttIndexBucket,
+ sizeProfile,
+ steadyRate,
+ summarizeDelayTail,
+ summarizeRttSamples,
+} from './97_bench_rtt';
export interface BenchStepTiming {
/** Date.now() at step body entry */
@@ -109,6 +140,9 @@ const SL_READY_NAMESPACE = 'bench-sl-ready';
// reader-ready barrier pattern SL uses.
const SO_STREAM_NAMESPACE = 'bench-so';
const SO_READY_NAMESPACE = 'bench-so-ready';
+// Dedicated streams for the CRTT scenario, same isolation + barrier pattern.
+const CRTT_STREAM_NAMESPACE = 'bench-crtt';
+const CRTT_READY_NAMESPACE = 'bench-crtt-ready';
// Deterministic, variable-length text fragments cycled to approximate real
// token-stream traffic (≈4.5 UTF-8 bytes on average, including punctuation and
// newline "tokens") while keeping every run byte-for-byte reproducible.
@@ -141,6 +175,68 @@ function soChunk(
: text;
}
+/** A self-timestamping CRTT chunk. `text` keeps the payload LLM-shaped (the
+ * same cycled fragments the SO scenarios stream); the `'sweep'` variant adds
+ * `pad` so the serialized chunk size rotates across the size buckets. */
+export interface BenchChunkRttDelta {
+ seq: number;
+ /** Date.now() in the writer step immediately before this chunk's write */
+ writtenAt: number;
+ text: string;
+ pad?: string;
+}
+
+/** CRTT payload variant. `'llm'` streams LLM-shaped deltas (a few tens of
+ * bytes each, so the index numbers stay pure of padding); `'sweep'` pads
+ * deltas in rotation across log-spaced sizes so mean RTT can be profiled as
+ * a function of serialized chunk size. (The replay scenario replays real
+ * captured cadences instead — see {@link benchReplayWorkflow}.) */
+export type BenchChunkRttVariant = 'llm' | 'sweep';
+
+/** Reader-side aggregation of one CRTT run: per-bucket summaries computed on
+ * the deployment (see 97_bench_rtt.ts). Buckets that received no samples are
+ * absent. */
+/** Chunk delay variation for one run, aggregated in the reader step (see
+ * computeCdv in 97_bench_rtt.ts for the definition and pairing rules). */
+export interface BenchChunkCdv {
+ /** Number of seq-adjacent pairs measured. */
+ pairs: number;
+ /** Adjacent arrivals whose seqs weren't consecutive (0 by contract). */
+ skippedPairs: number;
+ /** Tail of POSITIVE cdv — delivery clumps/stalls. Negatives (catch-up)
+ * balance them by the telescoping identity and are not summarized. */
+ positive?: BenchDelayTail;
+ /** Mean positive cdv per tenth of the stream — localizes where delivery
+ * clumping/stalls concentrate. */
+ progress: BenchRttMeanProfile;
+}
+
+export interface BenchChunkRttResult {
+ /** Number of chunks the reader received (validated against the request) */
+ received: number;
+ /** All chunks pooled — the headline "average per-chunk RTT" summary. */
+ all?: BenchRttSummary;
+ byIndex: Partial>;
+ /** Mean RTT per tenth of the stream — the drift/trend readout that fixed
+ * index buckets cannot provide (see progressProfile in 97_bench_rtt.ts). */
+ progress: BenchRttMeanProfile;
+ /** Mean RTT per log size bin — the size→latency curve (only informative
+ * for the `'sweep'` variant, whose pad rotation occupies every bin). */
+ size: BenchRttMeanProfile;
+ /** Chunk delay variation (delivery jitter), from RAW timestamps. */
+ cdv: BenchChunkCdv;
+ /** Delivered (reader-side) sustained throughput over the steady window. */
+ delivered?: BenchSteadyRate;
+}
+
+// Pad lengths cycled by the CRTT `'sweep'` variant: a log ladder chosen so
+// the ~60B base chunk serializes to one representative size per size-profile
+// bin (~160B, ~400B, ~760B, ~1.5KB, ~3KB, ~6KB, ~12KB — see
+// RTT_SIZE_BIN_EDGES_BYTES in 97_bench_rtt.ts). Rotation decouples size from
+// seq: every size appears throughout the stream, so the size profile is not
+// confounded with warmup or drift.
+const CRTT_SWEEP_PAD_LENGTHS = [100, 340, 700, 1400, 3000, 6000, 12000];
+
async function timedNoopStep(index: number): Promise {
'use step';
const kind = stepKind();
@@ -438,3 +534,324 @@ export async function benchSoWorkflow(
},
};
}
+
+/** Reader half of the CRTT scenario. Same attach/ready handshake as
+ * {@link soReaderStep}, but each received chunk is scored individually:
+ * `rtt = Date.now() - chunk.writtenAt` (clamped at 0 to absorb tiny
+ * intra-Vercel clock skew between the writer's and reader's instances) and
+ * an approximate serialized size (`JSON.stringify` length — the payloads are
+ * ASCII, so chars ≈ UTF-8 bytes). Raw (unclamped) timestamps are also kept
+ * in arrival order for chunk delay variation — the skew-free
+ * delivery-jitter companion metric (see computeCdv). Everything is
+ * aggregated here in the step, so the workflow returns compact summaries
+ * rather than one number per chunk. */
+async function crttReaderStep(): Promise {
+ 'use step';
+ const { workflowRunId } = getWorkflowMetadata();
+ const reader = getRun(workflowRunId)
+ .getReadable({ namespace: CRTT_STREAM_NAMESPACE })
+ .getReader();
+ try {
+ // Initiate the read BEFORE signalling ready so the stream GET is in flight
+ // by the time the writer starts (identical to the SL/SO handshake).
+ const firstRead = reader.read();
+
+ const ready = getWritable<{ ready: true }>({
+ namespace: CRTT_READY_NAMESPACE,
+ });
+ const readyWriter = ready.getWriter();
+ await readyWriter.write({ ready: true });
+ readyWriter.releaseLock();
+ await ready.close();
+
+ const all: number[] = [];
+ // RTT per seq (indexed by the chunk's own seq, not arrival order) so the
+ // progress profile bins by position in the stream even if delivery ever
+ // reorders.
+ const rttBySeq: (number | undefined)[] = [];
+ // RAW timestamps in arrival order for CDV — the clamped RTTs below must
+ // never feed it (clamping breaks cdv_i = CTT_i - CTT_{i-1} and hides the
+ // negative catch-up half of every clump). Bytes ride along for the
+ // delivered-throughput computation.
+ const arrivals: (CdvArrival & { bytes: number })[] = [];
+ const sizeSamples: { bytes: number; rttMs: number }[] = [];
+ const byIndex = new Map();
+ let received = 0;
+ let result = await firstRead;
+ while (!result.done) {
+ const receivedAt = Date.now();
+ const chunk = result.value;
+ if (
+ !chunk ||
+ typeof chunk.seq !== 'number' ||
+ typeof chunk.writtenAt !== 'number'
+ ) {
+ throw new Error(
+ `bench CRTT reader: malformed chunk ${JSON.stringify(chunk)?.slice(0, 120)}`
+ );
+ }
+ const rtt = Math.max(0, receivedAt - chunk.writtenAt);
+ all.push(rtt);
+ rttBySeq[chunk.seq] = rtt;
+ // Approximate serialized bytes (ASCII payloads, so chars ≈ bytes).
+ const bytes = JSON.stringify(chunk).length;
+ arrivals.push({
+ seq: chunk.seq,
+ writtenAt: chunk.writtenAt,
+ readAt: receivedAt,
+ // Extra field beyond CdvArrival — reused for delivered throughput.
+ bytes,
+ });
+ sizeSamples.push({ bytes, rttMs: rtt });
+ const bucket = rttIndexBucket(chunk.seq);
+ const samples = byIndex.get(bucket);
+ if (samples) samples.push(rtt);
+ else byIndex.set(bucket, [rtt]);
+ received++;
+ result = await reader.read();
+ }
+
+ const summarize = (buckets: Map) => {
+ const out: Partial> = {};
+ for (const [bucket, samples] of buckets) {
+ out[bucket] = summarizeRttSamples(samples);
+ }
+ return out;
+ };
+ const cdv = computeCdv(arrivals);
+ // Ordered, complete delivery is this bench's contract; a violation is a
+ // stream-integrity failure, not a latency data point. With no
+ // duplicates and no reorders, zero skipped pairs plus a first seq of 0
+ // makes the received sequence exactly contiguous 0..received-1 — the
+ // runner's received-count check alone can't distinguish a hole from a
+ // relabeled range.
+ if (
+ cdv.duplicateSeqs > 0 ||
+ cdv.reorderedArrivals > 0 ||
+ cdv.skippedPairs > 0 ||
+ (arrivals.length > 0 && arrivals[0].seq !== 0)
+ ) {
+ throw new Error(
+ `bench CRTT reader: stream integrity violated (duplicates=${cdv.duplicateSeqs}, reordered=${cdv.reorderedArrivals}, holes=${cdv.skippedPairs}, firstSeq=${arrivals[0]?.seq})`
+ );
+ }
+ return {
+ received,
+ all: summarizeRttSamples(all),
+ byIndex: summarize(byIndex),
+ progress: progressProfile(rttBySeq),
+ size: sizeProfile(sizeSamples),
+ cdv: {
+ pairs: cdv.cdvMs.length,
+ skippedPairs: cdv.skippedPairs,
+ positive: summarizeDelayTail(cdv.cdvMs.filter((v) => v > 0)),
+ progress: progressProfile(cdv.positiveBySeq),
+ },
+ delivered: steadyRate(
+ arrivals.map((a) => ({ atMs: a.readAt, bytes: a.bytes }))
+ ),
+ };
+ } finally {
+ reader.cancel().catch(() => {});
+ }
+}
+
+/** Writer half of the CRTT scenario: identical pacing to {@link soWriterStep}
+ * (ready barrier, then `chunkCount` chunks at one per `intervalMs`, writing
+ * immediately when behind schedule), but every chunk is self-timestamping —
+ * `writtenAt` is stamped immediately before its write — so the reader can
+ * compute a per-chunk RTT instead of a whole-stream span.
+ *
+ * Also reports write slip: `writtenAt_i - scheduledAt_i`, how late each write
+ * happened vs its open-loop schedule. This is the producer-stall guard
+ * per-chunk RTT structurally cannot provide — a write delayed by
+ * backpressure is stamped late, so its RTT still looks fine (coordinated
+ * omission), but its slip grows. For slip to mean anything the schedule MUST
+ * stay absolute from `startedAt` (as below): sleeping a fixed interval after
+ * each awaited write would re-anchor the schedule to the writes themselves
+ * (closed-loop) and hide the stall. */
+async function crttWriterStep(
+ chunkCount: number,
+ intervalMs: number,
+ variant: BenchChunkRttVariant
+): Promise<{ slip?: BenchDelayTail; achieved?: BenchSteadyRate }> {
+ 'use step';
+ const { workflowRunId } = getWorkflowMetadata();
+ const readyReader = getRun<{ ready: true }>(workflowRunId)
+ .getReadable<{ ready: true }>({ namespace: CRTT_READY_NAMESPACE })
+ .getReader();
+ try {
+ await readyReader.read();
+ } finally {
+ readyReader.cancel().catch(() => {});
+ }
+
+ const writable = getWritable({
+ namespace: CRTT_STREAM_NAMESPACE,
+ });
+ const writer = writable.getWriter();
+ const slips: number[] = [];
+ const writes: { atMs: number; bytes: number }[] = [];
+ const startedAt = Date.now();
+ for (let i = 0; i < chunkCount; i++) {
+ const scheduledAt = startedAt + (i + 1) * intervalMs;
+ const delay = scheduledAt - Date.now();
+ if (delay > 0) {
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ }
+ const chunk: BenchChunkRttDelta = {
+ seq: i,
+ writtenAt: Date.now(),
+ text: SO_TEXT_FRAGMENTS[i % SO_TEXT_FRAGMENTS.length],
+ };
+ if (variant === 'sweep') {
+ chunk.pad = 'x'.repeat(
+ CRTT_SWEEP_PAD_LENGTHS[i % CRTT_SWEEP_PAD_LENGTHS.length]
+ );
+ }
+ // Slip is stamped at the same instant as `writtenAt` (just before the
+ // write is enqueued); the awaited write's own duration surfaces in the
+ // NEXT chunk's slip when it pushes that chunk past its schedule.
+ slips.push(Math.max(0, chunk.writtenAt - scheduledAt));
+ writes.push({ atMs: chunk.writtenAt, bytes: JSON.stringify(chunk).length });
+ await writer.write(chunk);
+ }
+ writer.releaseLock();
+ await writable.close();
+ return {
+ slip: summarizeDelayTail(slips),
+ // Achieved (writer-side) sustained rate over the steady window: under
+ // healthy pacing ≈ the nominal rate; if writes block, this is what the
+ // producer actually managed.
+ achieved: steadyRate(writes),
+ };
+}
+
+/**
+ * Scenario 7: per-chunk round-trip time (CRTT), measured entirely on the
+ * deployment.
+ *
+ * Same shape as the SO scenario (paced writer + parallel draining reader on a
+ * dedicated namespaced stream, reader-ready barrier), but the measurement is
+ * per chunk rather than per stream: every delta embeds `{ seq, writtenAt }`
+ * (the SL scenario's payload-embedded-timestamp trick applied to all chunks),
+ * and the reader computes each chunk's write->read RTT on arrival (the "round
+ * trip" being deployment -> stream backend -> co-located reader, not an echo
+ * back to the writer).
+ *
+ * Naming: CRTT (chunk ROUND-trip time) is reserved for this same-clock-domain
+ * setup, where "round" is literally true — the chunk returns to the
+ * deployment whose clock stamped it. The future production write->read
+ * metric crosses clocks (producer deployment -> arbitrary consumer) and is a
+ * one-way trip: that one is CTT (chunk trip time), a separate metric with
+ * its own clock-skew caveats. Keep the names distinct. The reader aggregates the samples on the deployment
+ * (see 97_bench_rtt.ts): chunk-index buckets, a per-tenth-of-stream progress
+ * profile, a per-log-size-bin size profile, and fixed log-bin histograms so
+ * distributions merge and diff exactly across runs. The `'llm'` variant
+ * streams the same LLM-shaped deltas as SO (index/progress numbers pure of
+ * padding); the `'sweep'` variant pads deltas in rotation across log-spaced
+ * sizes (~160B to ~12KB serialized) so the size profile becomes a
+ * size->latency curve.
+ */
+export async function benchCrttWorkflow(
+ chunkCount: number,
+ intervalMs: number,
+ variant: BenchChunkRttVariant = 'llm'
+): Promise<{
+ crtt: BenchChunkRttResult;
+ writeSlip?: BenchDelayTail;
+ achieved?: BenchSteadyRate;
+}> {
+ 'use workflow';
+ const [crtt, writer] = await Promise.all([
+ crttReaderStep(),
+ crttWriterStep(chunkCount, intervalMs, variant),
+ ]);
+ return { crtt, writeSlip: writer.slip, achieved: writer.achieved };
+}
+
+/** Writer half of the replay scenario: identical structure to
+ * {@link crttWriterStep} (ready barrier, absolute open-loop schedule, slip +
+ * achieved-rate reporting), but the schedule and per-chunk sizes come from a
+ * REAL captured eve cadence (see 97_bench_cadence.ts) instead of a fixed
+ * interval: chunk i is scheduled at `startedAt + offsetsMs[i] / speed` and
+ * padded so its serialized size matches the capture. Missed ticks are never
+ * re-spread — overdue chunks write back-to-back and the lost time surfaces
+ * as slip (open-loop; see the crttWriterStep caveat). */
+async function replayWriterStep(
+ cadenceId: string,
+ speed: number
+): Promise<{ slip?: BenchDelayTail; achieved?: BenchSteadyRate }> {
+ 'use step';
+ const cadence = BENCH_CADENCES[cadenceId];
+ if (!cadence) {
+ throw new Error(`bench replay writer: unknown cadence "${cadenceId}"`);
+ }
+ const { workflowRunId } = getWorkflowMetadata();
+ const readyReader = getRun<{ ready: true }>(workflowRunId)
+ .getReadable<{ ready: true }>({ namespace: CRTT_READY_NAMESPACE })
+ .getReader();
+ try {
+ await readyReader.read();
+ } finally {
+ readyReader.cancel().catch(() => {});
+ }
+
+ const writable = getWritable({
+ namespace: CRTT_STREAM_NAMESPACE,
+ });
+ const writer = writable.getWriter();
+ const slips: number[] = [];
+ const writes: { atMs: number; bytes: number }[] = [];
+ const startedAt = Date.now();
+ for (let i = 0; i < cadence.offsetsMs.length; i++) {
+ const scheduledAt = startedAt + cadence.offsetsMs[i] / speed;
+ const delay = scheduledAt - Date.now();
+ if (delay > 0) {
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ }
+ const chunk: BenchChunkRttDelta = {
+ seq: i,
+ writtenAt: Date.now(),
+ text: SO_TEXT_FRAGMENTS[i % SO_TEXT_FRAGMENTS.length],
+ };
+ // Pad the delta so its serialized size matches the captured event's
+ // (~60B envelope of seq/writtenAt/text; exactness beyond a few bytes
+ // doesn't matter — the doubling size bins absorb it).
+ const pad = cadence.sizes[i] - 60;
+ if (pad > 0) chunk.pad = 'x'.repeat(pad);
+ slips.push(Math.max(0, chunk.writtenAt - scheduledAt));
+ writes.push({ atMs: chunk.writtenAt, bytes: JSON.stringify(chunk).length });
+ await writer.write(chunk);
+ }
+ writer.releaseLock();
+ await writable.close();
+ return { slip: summarizeDelayTail(slips), achieved: steadyRate(writes) };
+}
+
+/**
+ * Scenario 8: cadence replay, measured entirely on the deployment.
+ *
+ * Same reader, barrier, and measurement machinery as {@link benchCrttWorkflow}
+ * (per-chunk CRTT, CDV, profiles, delivered rate), but the writer replays a
+ * REAL captured eve stream cadence at `speed`x: every write instant and every
+ * chunk size comes from the capture, so nothing about the workload shape is a
+ * judgment call except the speed multiplier. Eve's protocol re-ships the
+ * cumulative message per delta, so sizes ramp through the turn — the
+ * end-of-turn byte-rate peak is part of the workload, not an accident.
+ */
+export async function benchReplayWorkflow(
+ cadenceId: string,
+ speed: number
+): Promise<{
+ crtt: BenchChunkRttResult;
+ writeSlip?: BenchDelayTail;
+ achieved?: BenchSteadyRate;
+}> {
+ 'use workflow';
+ const [crtt, writer] = await Promise.all([
+ crttReaderStep(),
+ replayWriterStep(cadenceId, speed),
+ ]);
+ return { crtt, writeSlip: writer.slip, achieved: writer.achieved };
+}
diff --git a/workbench/example/workflows/97_bench_cadence.ts b/workbench/example/workflows/97_bench_cadence.ts
new file mode 100644
index 0000000000..f6bd4be17c
--- /dev/null
+++ b/workbench/example/workflows/97_bench_cadence.ts
@@ -0,0 +1,75 @@
+// Real captured stream cadences for the replay scenarios.
+//
+// GENERATED from eve-workflow-stream-replay v1 captures (slimmed to
+// per-event {offsetMs, serialized size} + provenance); sources at
+// ~/journal/playgrounds/durable-streams/replays/.cadence.json. Do not
+// hand-edit; re-capture and regenerate. A re-capture is a NEW workload —
+// the id is part of the baseline key, so bump it (new capture file name).
+// Re-capture requires the trace-lab plumbing vendored (with restore docs)
+// at ~/journal/playgrounds/workflow-server-profiling/eve-plumbing/;
+// captures from that setup record dirty:true, meaning exactly that
+// plumbing.
+//
+// Boundary prefixes are load-bearing: eve-* = eve's envelope-protocol
+// workload (message-so-far re-ships the cumulative message; sizes RAMP
+// 142B → 13KB; ~50x raw delta bytes; the demanding outlier tenant).
+// gateway-* = raw provider SSE deltas, no envelope (p50 208B = modal
+// production chunk size; the typical customer). -2000t = target
+// output-token anchor (~production p50 turn length); metadata carries
+// actuals.
+
+/** One captured cadence: when each chunk was written (offset from the first
+ * write) and how many bytes it serialized to. */
+export interface BenchCadence {
+ id: string;
+ /** Which boundary the capture was taken at: eve = envelope-protocol
+ * workload, gateway = raw provider SSE chunks. */
+ boundary: 'eve' | 'gateway';
+ model: string;
+ capturedAt: string;
+ /** Version of the eve capture tooling (for gateway captures this is the
+ * probe's eve version, not a workload property). */
+ eveVersion: string;
+ eveCommit: string;
+ events: number;
+ spanMs: number;
+ totalBytes: number;
+ offsetsMs: number[];
+ sizes: number[];
+}
+
+export const BENCH_CADENCES: Record = {
+ // eveCommit '(dirty)': verified fixture-wiring-only dirt (zero
+ // modifications under packages/eve/src); cadence validity vs 0.30.0 also
+ // checked by emission diff audit + an empirical nano re-capture.
+ 'eve-gpt-5.6-sol-2000t': {
+ id: 'eve-gpt-5.6-sol-2000t',
+ boundary: 'eve',
+ model: 'openai/gpt-5.6-sol',
+ capturedAt: '2026-08-12T20:00:42.830Z',
+ eveVersion: '0.33.3',
+ eveCommit: 'ebebdc059345 (dirty)',
+ events: 2593,
+ spanMs: 52377,
+ totalBytes: 17144887,
+ // biome-ignore format: generated capture data
+ offsetsMs: [0,0,1,1,4501,4501,4502,4537,4547,4548,4557,4557,4557,4596,4605,4736,4746,4746,4746,4746,4746,4793,4840,4856,4865,4865,4875,4875,4898,4916,4961,4971,5031,5042,5051,5062,5062,5062,5062,5088,5098,5152,5165,5174,5218,5232,5242,5242,5242,5242,5279,5288,5298,5298,5344,5354,5364,5364,5374,5374,5384,5384,5403,5423,5430,5430,5470,5481,5492,5492,5534,5544,5554,5564,5565,5598,5615,5625,5635,5635,5635,5635,5681,5683,5721,5735,5746,5746,5785,5795,5848,5859,5868,5879,5879,5879,5879,5933,5943,5943,5943,5943,5943,5943,5975,5985,5985,5995,6005,6005,6005,6005,6036,6060,6072,6072,6072,6072,6072,6118,6124,6125,6125,6135,6135,6135,6135,6210,6278,6288,6339,6351,6361,6401,6478,6479,6536,6539,6549,6588,6599,6650,6668,6678,6678,6678,6678,6713,6780,6816,6826,6826,6826,6857,6867,6867,6867,6903,6912,6923,6933,6934,6934,6962,6980,6990,6990,7025,7034,7087,7098,7149,7160,7171,7171,7180,7180,7180,7180,7207,7225,7238,7247,7247,7247,7247,7271,7281,7281,7291,7291,7301,7301,7330,7403,7412,7460,7470,7515,7590,7600,7601,7601,7637,7650,7659,7659,7669,7669,7669,7699,7719,7726,7726,7761,7775,7786,7834,7844,7844,7895,7914,7923,7924,7924,7924,7943,8003,8013,8064,8125,8185,8197,8214,8245,8260,8266,8266,8314,8322,8322,8366,8426,8436,8446,8456,8456,8485,8509,8519,8519,8519,8519,8519,8550,8565,8573,8605,8625,8634,8634,8634,8663,8726,8739,8749,8782,8807,8817,8817,8842,8919,8966,9021,9047,9057,9057,9057,9057,9082,9141,9203,9214,9261,9322,9373,9383,9393,9393,9393,9414,9464,9465,9500,9557,9569,9616,9630,9640,9677,9741,9802,9811,9859,9878,9884,9920,9939,9949,9949,9949,9949,9985,9995,9995,10039,10054,10065,10065,10101,10111,10193,10217,10279,10295,10302,10302,10302,10338,10403,10413,10459,10486,10520,10533,10544,10578,10597,10606,10607,10607,10607,10643,10701,10726,10738,10748,10748,10748,10748,10763,10774,10784,10830,10840,10840,10841,10841,10841,10841,10851,10860,10861,10861,10861,10885,10895,10895,10938,10956,10967,10967,11006,11017,11017,11059,11072,11083,11123,11133,11145,11155,11155,11186,11197,11207,11217,11228,11228,11228,11228,11248,11320,11330,11376,11387,11387,11387,11435,11447,11457,11467,11467,11467,11501,11516,11526,11536,11536,11536,11536,11567,11577,11587,11587,11629,11692,11703,11759,11785,11795,11795,11825,11834,11835,11884,11888,11937,12012,12033,12043,12043,12060,12093,12103,12103,12103,12117,12192,12203,12212,12213,12213,12213,12213,12239,12250,12304,12369,12482,12482,12483,12491,12504,12512,12512,12522,12522,12532,12532,12553,12568,12580,12593,12600,12600,12613,12672,12725,12741,12751,12789,12803,12814,12823,12823,12823,12823,12846,12858,12905,12973,12983,13025,13037,13048,13078,13078,13078,13078,13078,13090,13099,13156,13166,13167,13211,13215,13270,13287,13297,13297,13297,13298,13298,13326,13343,13351,13351,13361,13361,13361,13390,13467,13469,13469,13477,13477,13503,13519,13529,13529,13539,13540,13540,13540,13564,13574,13622,13632,13682,13747,13769,13769,13819,13822,13841,13851,13851,13862,13887,13916,13975,13994,14004,14042,14054,14054,14054,14123,14124,14124,14126,14126,14126,14126,14162,14174,14174,14185,14185,14185,14185,14218,14228,14274,14288,14303,14314,14314,14314,14314,14334,14387,14408,14419,14419,14454,14533,14534,14534,14534,14541,14542,14542,14542,14571,14587,14587,14642,14702,14702,14702,14703,14714,14714,14744,14760,14770,14780,14780,14780,14780,14801,14915,14917,14917,14934,14945,14949,14949,14949,14949,14987,14999,15009,15020,15020,15020,15021,15038,15096,15161,15215,15224,15238,15272,15342,15351,15367,15376,15376,15376,15376,15391,15404,15414,15508,15528,15532,15540,15551,15551,15552,15552,15593,15608,15615,15615,15615,15615,15615,15637,15664,15674,15675,15684,15694,15694,15694,15710,15720,15730,15730,15730,15731,15752,15763,15775,15785,15785,15785,15786,15806,15873,15877,15892,15902,15902,15902,15902,15926,15940,15950,15960,15960,15960,15960,15982,15993,16003,16049,16066,16076,16076,16076,16106,16120,16131,16140,16141,16141,16141,16162,16175,16221,16234,16244,16244,16254,16254,16254,16254,16275,16299,16340,16358,16367,16368,16403,16430,16455,16501,16512,16512,16512,16541,16551,16551,16551,16551,16551,16551,16568,16628,16648,16659,16660,16682,16691,16743,16753,16763,16763,16800,16820,16829,16830,16830,16868,16897,16907,16907,16907,17380,17390,17390,17391,17391,17391,17391,17391,17391,17391,17391,17391,17392,17392,17392,17411,17421,17421,17421,17515,17526,17526,17526,17535,17564,17578,17626,17639,17649,17683,17695,17706,17716,17716,17730,17730,17730,17740,17751,17798,17865,17914,17926,17972,18001,18012,18033,18047,18056,18056,18089,18149,18254,18320,18335,18345,18345,18378,18389,18402,18412,18412,18413,18413,18480,18481,18481,18481,18495,19433,19452,19463,19473,19473,19473,19473,19488,19555,19563,19573,19585,19597,19597,19610,19628,19643,19652,19652,19652,19672,19698,19708,19708,19709,19709,19745,19793,19812,19822,19823,19823,19861,19871,19886,19892,19902,19902,19921,19931,19941,19941,19986,20044,20053,20101,20118,20128,20128,20128,20162,20175,20185,20195,20195,20234,20249,20249,20249,20254,20255,20255,20255,20286,20314,20316,20316,20316,20316,20351,20365,20365,20371,20411,20433,20443,20443,20443,20443,20453,20453,20469,20497,20507,20507,20507,20508,20508,20532,20541,20625,20652,20664,20672,20683,20683,20683,20693,20693,20715,20767,20787,20797,20797,20825,20839,20885,20902,20913,20956,20966,20966,20966,20966,21002,21017,21032,21032,21037,21037,21037,21037,21061,21075,21084,21126,21134,21183,21195,21210,21220,21220,21220,21220,21243,21252,21252,21263,21263,21263,21305,21319,21329,21329,21363,21383,21393,21393,21394,21394,21419,21430,21478,21490,21538,21550,21565,21575,21575,21575,21575,21606,21628,21657,21671,21681,21717,21793,21801,21801,21801,21811,21822,21832,21832,21851,21861,21862,21873,21873,21874,21874,21938,21955,22098,22109,22109,22109,22110,22110,22110,22110,22110,22110,22110,22110,22111,22210,22221,22221,22221,22221,22221,22221,22394,22404,22404,22404,22404,22405,22405,22405,22405,22414,22497,22497,22498,22498,22498,22498,22512,22572,22583,22630,22641,22698,22753,22806,22823,22831,22831,22831,22831,22864,22875,22890,22906,22906,22906,22906,22923,22937,22981,23002,23011,23011,23011,23046,23062,23114,23172,23232,23242,23302,23323,23334,23334,23334,23334,23355,23415,23494,23508,23534,23544,23600,23612,23648,23713,23768,23822,23880,23890,23906,23914,23914,23914,23914,23940,23951,23961,23996,24058,24077,24087,24087,24113,24123,24171,24228,24284,24297,24341,24404,24414,24414,24455,24516,24524,24535,24569,24582,24592,24626,24638,24683,24699,24709,24709,24736,24795,24805,24859,24869,24908,24968,24978,24988,24998,24998,24998,24998,25019,25040,25050,25050,25092,25102,25139,25177,25188,25202,25245,25314,25328,25367,25380,25390,25390,25390,25390,25400,25417,25536,25537,25537,25537,25537,25537,25537,25537,25550,25552,25561,25600,25610,25660,25670,25680,25680,25713,25724,25734,25744,25744,25744,25744,25770,25788,25797,25797,25835,25847,25856,25857,25867,25867,25867,25867,25887,25905,25915,25950,25998,26008,26008,26008,26008,26017,26017,26018,26064,26119,26177,26235,26245,26255,26298,26308,26308,26351,26409,26429,26437,26447,26447,26447,26447,26465,26476,26528,26545,26582,26592,26641,26665,26674,26684,26685,26685,26685,26711,26725,26735,26746,26746,26746,26746,27421,27439,27451,27451,27451,27451,27484,27496,27561,27634,27644,27644,27644,27644,27688,27698,27698,27698,27698,27728,27796,27806,27806,27807,27807,27807,27807,27807,27929,27944,27960,27962,27963,27963,27969,27979,28225,28237,28237,28237,28237,28238,28238,28238,28238,28238,28238,28238,28238,28238,28238,28238,28238,28327,28340,28344,28383,28393,28393,28393,28419,28463,28484,28520,28533,28542,28542,28578,28596,28606,28649,28702,28718,28728,28738,28738,28738,28738,28759,28827,28844,28855,28855,28882,28901,28911,28911,28954,28964,28974,28974,28974,28974,29007,29074,29083,29129,29280,29290,29290,29290,29299,29300,29315,29315,29315,29315,29323,29332,29332,29366,29379,29402,29412,29412,29412,29412,29557,29558,29559,29560,29560,29560,29564,29576,29577,29577,29634,29636,29732,29743,29753,29753,29753,29793,29803,29855,29872,29881,29881,29881,29881,29915,29937,29973,29984,29995,30005,30006,30006,30006,30033,30049,30059,30059,30059,30095,30160,30170,30180,30233,30234,30272,30273,30277,30285,30350,30352,30392,30393,30409,30411,30459,30459,30460,30460,30460,30517,30529,30539,30539,30578,30589,30599,30641,30651,30671,30701,30714,30724,30724,30738,30751,30751,30762,30770,30817,30845,30855,30863,30863,30863,30863,30881,30891,30903,30903,30903,30940,30954,30977,30985,30985,30985,30997,31016,31025,31025,31059,31070,31114,31125,31134,31134,31134,31144,31144,31145,31145,31180,31191,31215,31226,31226,31226,31226,31239,31297,31363,31368,31422,31432,31443,31480,31493,31501,31501,31540,31547,31557,31596,31615,31623,31624,31698,31716,31725,31726,31726,31768,31798,31809,31810,31810,31833,31883,31896,31904,31913,31923,31923,31924,31924,31943,31956,31972,31979,31979,32000,32013,32023,32059,32075,32086,32119,32135,32147,32158,32158,32158,32183,32202,32238,32257,32266,32306,32324,32325,32331,32336,32336,32336,32337,32356,32423,32433,32509,32509,32510,32510,32510,32510,32510,32536,32551,32573,32577,32577,32577,32577,32602,32612,32677,32687,32699,32699,32699,32699,32716,32780,32839,32897,32923,32934,32934,32934,32934,32934,32964,32975,32975,32975,33032,33041,33041,33070,33092,33102,33102,33134,33144,33191,33213,33223,33223,33233,33233,33233,33233,33247,33259,33272,33282,33282,33306,33319,33329,33329,33340,33340,33363,33380,33390,33400,33400,33400,33400,33440,33450,33478,33505,33516,33516,33516,33516,33516,33544,33554,33555,34317,34375,34389,34399,34438,34503,34511,34521,34534,34544,34545,34545,34557,34616,34693,34696,34712,34722,34723,34723,34750,34761,34761,34761,34771,34771,34771,34771,34796,34808,34821,34831,34831,34859,34871,34884,34897,34897,34897,34917,34978,34997,35007,35017,35017,35017,35038,35049,35060,35099,35109,35119,35119,35119,35162,35172,35172,35183,35193,35193,35193,35193,35217,35230,35241,35250,35260,35260,35261,35277,35301,35311,35311,35311,35311,35339,35351,35361,35361,35399,35509,35509,35509,35533,35543,35553,35553,35553,35553,35553,35591,35600,35601,35611,35612,35612,35612,35642,35652,35662,35672,35673,35673,35673,35703,35713,35713,36717,36863,36881,36893,36894,36894,36894,36894,36946,36967,36977,36991,37020,37037,37047,37047,37057,37069,37079,37079,37089,37089,37127,37200,37234,37259,37275,37286,37295,37305,37316,37325,37357,37369,37369,37369,37382,37404,37422,37422,37422,37422,37422,37426,37426,37427,37427,37427,37427,37427,37427,37427,37436,37436,37445,37457,37457,37465,37465,37465,37466,37475,37475,37475,37475,37475,37475,37475,37498,37508,37508,37508,37582,37592,37593,37593,37594,37594,37594,37624,37634,37706,37709,37719,37719,37720,37720,37774,37784,37784,37785,37785,37795,37795,37811,37821,37821,37831,37841,37842,37842,37842,37880,37890,37900,37910,37910,37911,37911,37947,37959,37969,37979,37989,38013,38039,38053,38063,38073,38074,38074,38074,38084,38121,38131,38331,38333,38342,38356,38362,38363,38363,38375,38386,38386,38386,38411,38423,38423,38423,38423,38423,38423,38424,38444,38464,38475,38475,38475,38475,38475,38497,38507,38507,38507,38552,38614,38624,38673,38688,38698,38698,38698,38698,38727,38795,38805,38805,38814,38815,38815,38815,38857,38866,38867,38867,38867,38867,38867,38910,38932,38932,38961,38971,39021,39039,39049,39049,39049,39049,39049,39074,39153,39173,39183,39183,39183,39183,39183,39192,39193,39245,39308,39318,39328,39328,39328,39329,39373,39375,39385,39385,39385,39430,39540,39542,39543,39543,39543,39544,39544,39545,39547,39557,39647,39661,39670,39680,39690,39719,39751,39798,39822,39837,39858,39877,39887,39888,39897,39898,39898,39908,39908,39908,39908,39920,39920,39920,39920,39920,39920,39921,39921,39921,39921,39921,39921,39973,40000,40077,40087,40087,40097,40098,40098,40098,40141,40151,40151,40151,40151,40151,40181,40193,40203,40203,40240,40300,40310,40310,40354,40366,40366,40374,40411,40423,40434,40434,40470,40480,40490,40491,40543,40553,40553,40587,40597,40677,40687,40687,40697,40697,40707,40707,40717,40727,40737,40737,40737,40738,40777,40788,40788,40788,40788,40814,41016,41026,41035,41046,41069,41079,41095,41122,41274,41284,41284,41285,41285,41285,41285,41286,41286,41286,41286,41286,41368,41378,41378,41379,41379,41379,41379,41379,41379,41379,41388,41398,41399,41468,41478,41479,41479,41479,41479,41479,41501,41511,41561,41571,41620,41625,41625,41678,41699,41709,41709,41738,41747,41787,41805,41816,41828,41828,41828,41851,41866,41874,41884,41894,41894,41904,41904,41925,41935,41935,41936,41936,41958,42024,42054,42064,42064,42086,42096,42121,42131,42132,42132,42132,42141,42208,42221,42233,42242,42242,42242,42242,42277,42307,42346,42358,42368,42376,42401,42411,42411,42411,42412,42529,42531,42531,42531,42532,42532,42532,42546,42546,42546,42546,42556,42567,42567,42623,42637,42644,42644,42645,42645,42695,42705,42705,42715,42715,42716,42716,42750,42760,42770,42770,42770,42810,42822,42830,42830,42830,42830,42830,42883,42892,42892,42892,42926,42936,42936,42936,42936,42982,43044,43054,43109,43117,43174,43185,43197,43207,43207,43207,43234,43244,43244,43244,43290,43349,43413,43424,43469,43535,43545,43661,43670,43683,43693,43693,43750,43766,43776,43841,43860,43871,43889,43903,43912,43922,43922,43923,43923,44899,44899,44899,44900,44900,44946,44957,44957,44957,44992,45049,45059,45060,45109,45119,45119,45169,45182,45193,45193,45193,45228,45239,45286,45363,45373,45373,45374,45441,45448,45448,45449,45449,45449,45449,45476,45539,45550,45598,45607,45674,45745,45756,45756,45756,45756,45788,45855,45924,45933,45934,45934,45985,46040,46122,46132,46132,46132,46177,46187,46187,46197,46197,46197,46198,46232,46240,46240,46291,46302,46391,46401,46497,46497,46497,46519,46529,46546,46556,46556,46556,46556,46584,46594,46594,46594,46595,46630,46641,46651,46661,46661,46661,46661,46695,46705,46750,46806,46816,46826,46826,46836,46836,46836,46868,46878,46928,46938,46938,46938,46999,47003,47004,47004,47013,47014,47014,47048,47058,47058,47058,47098,47160,47216,47278,47349,47360,47360,47398,47408,47489,47501,47501,47524,47534,47583,47593,47648,47659,47714,47722,47734,47734,47734,47734,47763,47773,47783,47783,47783,47854,47864,47864,47864,47864,47897,47907,47947,47956,48004,48013,48013,48063,48073,48073,48130,48140,48140,48150,48150,48150,48182,48244,48254,48307,48356,48367,48412,48482,48492,48572,48583,48595,48659,48669,48669,48726,48782,48819,48822,48822,48839,48856,48867,48867,48896,48906,48961,48971,48971,49020,49030,49074,49084,49133,49143,49203,49253,49264,49264,49327,49338,49338,49338,49338,49377,49387,49387,49387,49431,49495,49497,49508,49508,49568,49578,49629,49639,49639,49639,49668,49728,49791,49808,49816,49816,49816,49849,49904,49962,50028,50088,50098,50098,50110,50110,50110,50110,50138,50215,50225,50225,50225,50225,50260,50327,50337,50337,50337,50380,50436,50500,50557,50617,50627,50682,50692,50692,50741,50749,50749,50811,50863,50983,50993,50993,51060,51069,51070,51099,51109,51119,51119,51119,51162,51172,51214,51236,51247,51257,51258,51258,51258,51272,51290,51300,51301,51301,51352,51362,51362,51415,51424,51425,51440,51450,51450,51450,51451,51483,51532,51542,51542,51542,51568,51632,51643,51661,51697,51706,51718,51727,51727,51736,51760,51770,51770,51770,51807,51886,51946,51957,51984,52062,52072,52330,52331,52377,52377],
+ // biome-ignore format: generated capture data
+ sizes: [218,142,739,187,198,205,211,218,228,228,230,249,254,266,264,270,271,274,277,296,289,301,302,312,315,314,328,322,329,341,339,354,355,360,371,373,370,375,396,388,404,399,411,414,435,430,446,446,443,462,470,467,472,476,486,481,484,486,488,494,492,497,502,510,520,516,532,533,534,541,562,557,569,563,574,572,584,589,593,610,613,612,611,616,625,623,629,639,634,641,657,661,662,672,671,678,693,689,696,695,699,708,704,715,719,716,721,722,722,727,730,733,738,736,737,742,743,755,750,753,755,761,761,769,770,789,792,791,800,814,810,819,819,822,833,842,844,845,848,853,851,868,891,893,890,893,902,898,905,919,914,932,934,930,943,946,943,950,966,970,987,981,996,999,1013,1019,1029,1022,1037,1037,1051,1052,1057,1074,1065,1070,1075,1083,1083,1086,1089,1100,1113,1109,1112,1133,1127,1130,1135,1137,1142,1156,1151,1151,1162,1172,1182,1179,1194,1213,1205,1208,1211,1218,1231,1224,1243,1243,1256,1253,1256,1261,1259,1268,1267,1273,1291,1296,1294,1311,1309,1319,1334,1339,1333,1348,1352,1364,1369,1372,1383,1387,1389,1401,1401,1413,1415,1412,1425,1432,1437,1441,1438,1459,1460,1456,1465,1476,1482,1477,1492,1496,1512,1503,1518,1526,1534,1536,1543,1547,1552,1563,1557,1562,1575,1579,1575,1588,1603,1604,1616,1615,1633,1648,1649,1648,1651,1662,1677,1676,1677,1694,1698,1699,1713,1718,1721,1734,1731,1730,1733,1745,1758,1768,1766,1788,1785,1792,1806,1808,1820,1833,1840,1833,1842,1843,1856,1862,1877,1868,1873,1886,1886,1892,1897,1898,1917,1920,1919,1924,1936,1932,1940,1947,1943,1948,1959,1976,1974,1971,1978,1978,1986,1991,1998,2002,2007,2010,2010,2010,2013,2015,2017,2027,2047,2040,2047,2046,2050,2059,2055,2068,2071,2068,2073,2082,2082,2093,2094,2092,2093,2098,2099,2099,2102,2102,2107,2108,2116,2115,2125,2135,2131,2139,2136,2143,2153,2166,2168,2166,2186,2189,2188,2197,2197,2206,2206,2211,2217,2220,2222,2227,2233,2240,2250,2246,2254,2253,2257,2258,2258,2261,2265,2270,2268,2269,2274,2283,2297,2289,2304,2304,2304,2306,2314,2311,2338,2340,2346,2351,2353,2357,2361,2367,2364,2367,2375,2384,2391,2390,2393,2404,2407,2414,2414,2433,2428,2439,2438,2441,2454,2456,2455,2475,2472,2470,2477,2495,2498,2507,2517,2528,2524,2531,2544,2537,2542,2559,2563,2566,2567,2570,2581,2596,2599,2598,2599,2601,2603,2613,2615,2621,2622,2635,2641,2640,2645,2649,2647,2654,2664,2677,2685,2690,2689,2702,2708,2707,2732,2726,2730,2728,2743,2741,2748,2755,2763,2764,2790,2789,2792,2801,2802,2810,2814,2821,2835,2833,2843,2852,2846,2853,2859,2866,2886,2889,2883,2890,2900,2913,2907,2907,2912,2911,2916,2941,2935,2939,2937,2942,2951,2961,2959,2982,2971,2980,2989,2990,3000,2999,3012,3018,3025,3028,3031,3037,3036,3040,3039,3044,3043,3044,3049,3048,3049,3054,3053,3054,3059,3058,3061,3065,3072,3071,3073,3081,3084,3086,3099,3105,3104,3107,3112,3110,3117,3137,3137,3147,3157,3150,3153,3153,3154,3157,3157,3158,3161,3161,3168,3166,3169,3175,3173,3180,3184,3188,3186,3193,3219,3220,3220,3226,3241,3249,3254,3249,3256,3260,3274,3283,3280,3301,3303,3304,3312,3316,3321,3324,3332,3346,3352,3350,3363,3367,3369,3379,3382,3381,3405,3408,3410,3419,3420,3426,3429,3445,3453,3446,3451,3462,3479,3481,3486,3486,3505,3510,3504,3509,3514,3520,3539,3533,3538,3546,3560,3556,3557,3561,3560,3567,3575,3575,3584,3590,3606,3611,3605,3610,3611,3611,3616,3617,3625,3628,3628,3634,3640,3647,3647,3646,3655,3652,3657,3660,3661,3665,3670,3678,3694,3703,3703,3706,3724,3719,3734,3727,3734,3740,3739,3743,3742,3749,3759,3758,3763,3775,3781,3780,3787,3788,3799,3794,3811,3808,3817,3821,3826,3835,3830,3839,3838,3852,3854,3855,3860,3864,3862,3871,3882,3880,3883,3886,3885,3900,3898,3911,3925,3922,3936,3943,3939,3953,3959,3968,3966,3983,3993,3995,3999,3996,4001,4002,4010,4019,4028,4026,4029,4030,4034,4037,4048,4069,4079,4103,4093,4105,4115,4114,4128,4137,4130,4135,4142,4141,4141,4146,4145,4154,4155,4156,4166,4176,4182,4177,4204,4198,4202,4200,4207,4219,4221,4228,4224,4237,4246,4248,4245,4252,4264,4274,4281,4283,4288,4309,4306,4319,4320,4333,4334,4349,4345,4362,4360,4358,4367,4366,4374,4381,4395,4393,4397,4395,4402,4406,4410,4422,4418,4422,4421,4432,4432,4443,4442,4445,4456,4469,4463,4463,4472,4473,4484,4485,4503,4510,4511,4516,4533,4524,4539,4539,4545,4560,4554,4554,4559,4558,4563,4588,4582,4586,4590,4594,4604,4607,4620,4617,4624,4621,4634,4635,4633,4638,4649,4648,4661,4663,4668,4667,4671,4672,4678,4676,4677,4682,4681,4686,4691,4697,4698,4703,4701,4702,4707,4706,4713,4713,4719,4717,4718,4721,4721,4722,4725,4727,4733,4731,4732,4737,4742,4752,4749,4751,4759,4768,4769,4777,4777,4792,4789,4812,4801,4808,4828,4838,4839,4861,4855,4859,4857,4866,4865,4877,4882,4890,4889,4896,4895,4899,4900,4904,4903,4906,4910,4909,4912,4916,4915,4918,4922,4921,4924,4928,4929,4933,4934,4936,4952,4949,4954,4956,4959,4966,4979,4974,4988,4986,4991,4989,4998,5001,5005,5017,5031,5036,5038,5035,5040,5055,5052,5061,5061,5068,5081,5078,5091,5099,5099,5113,5118,5117,5128,5131,5144,5151,5149,5156,5161,5160,5164,5165,5171,5169,5174,5173,5182,5183,5188,5194,5193,5195,5199,5208,5208,5213,5223,5218,5239,5232,5232,5237,5236,5237,5238,5241,5253,5252,5253,5261,5272,5266,5283,5300,5307,5307,5336,5327,5331,5345,5344,5344,5348,5363,5356,5363,5375,5381,5386,5400,5392,5403,5413,5415,5432,5427,5442,5443,5448,5457,5464,5469,5473,5470,5473,5473,5480,5490,5503,5501,5515,5528,5519,5528,5533,5540,5552,5549,5562,5572,5575,5571,5582,5596,5596,5613,5610,5632,5625,5634,5644,5652,5646,5657,5667,5681,5678,5676,5683,5701,5692,5701,5708,5708,5715,5722,5730,5739,5735,5743,5750,5746,5755,5758,5760,5777,5769,5784,5778,5785,5797,5791,5800,5805,5812,5816,5827,5827,5825,5840,5840,5854,5855,5872,5869,5873,5887,5890,5896,5904,5907,5912,5909,5912,5912,5923,5927,5926,5934,5945,5943,5948,5950,5965,5972,5968,5982,5990,5996,5999,6002,6014,6020,6025,6035,6033,6040,6043,6057,6062,6059,6071,6082,6081,6083,6088,6102,6109,6119,6112,6113,6114,6115,6116,6117,6118,6119,6120,6121,6122,6123,6132,6129,6144,6152,6146,6147,6154,6160,6157,6164,6168,6166,6173,6191,6182,6187,6196,6218,6220,6214,6215,6224,6237,6242,6251,6251,6269,6288,6283,6286,6298,6306,6300,6301,6302,6307,6312,6314,6323,6335,6340,6335,6340,6359,6360,6360,6365,6383,6382,6383,6392,6402,6410,6404,6407,6417,6419,6421,6436,6435,6433,6436,6444,6449,6464,6466,6480,6480,6477,6492,6492,6504,6506,6509,6521,6537,6535,6538,6550,6550,6552,6573,6563,6578,6586,6586,6594,6592,6592,6601,6602,6605,6610,6622,6620,6621,6629,6630,6641,6640,6649,6663,6663,6664,6665,6687,6679,6703,6696,6707,6722,6719,6753,6742,6740,6749,6754,6765,6765,6771,6768,6783,6783,6795,6809,6814,6830,6834,6842,6836,6847,6853,6865,6866,6875,6878,6877,6891,6893,6908,6910,6914,6931,6930,6931,6948,6958,6960,6956,6961,6964,6963,6968,6971,6974,6989,6992,6996,7005,7014,7032,7021,7024,7028,7039,7040,7050,7049,7078,7076,7079,7098,7096,7113,7110,7108,7119,7123,7132,7143,7142,7146,7158,7156,7159,7170,7173,7174,7179,7193,7188,7188,7199,7213,7211,7209,7220,7222,7228,7227,7235,7238,7244,7259,7263,7274,7267,7278,7290,7297,7297,7311,7310,7326,7324,7340,7338,7336,7337,7344,7350,7351,7368,7370,7382,7383,7380,7387,7393,7392,7392,7397,7396,7403,7407,7411,7413,7418,7422,7430,7436,7442,7444,7456,7476,7472,7477,7476,7490,7490,7496,7493,7504,7510,7524,7522,7520,7527,7537,7558,7554,7551,7552,7561,7566,7563,7570,7568,7573,7580,7583,7595,7601,7604,7605,7604,7605,7614,7615,7620,7624,7642,7647,7655,7653,7652,7653,7662,7661,7661,7666,7665,7688,7682,7683,7687,7686,7697,7693,7694,7701,7709,7709,7718,7724,7726,7738,7738,7742,7750,7762,7763,7780,7779,7782,7794,7794,7804,7817,7815,7813,7814,7821,7827,7844,7853,7851,7849,7858,7861,7875,7870,7870,7875,7874,7879,7892,7892,7896,7910,7905,7911,7923,7921,7938,7930,7939,7942,7946,7952,7953,7962,7962,7977,7976,7980,7988,7994,7996,8008,8016,8014,8029,8030,8033,8035,8040,8052,8054,8063,8064,8076,8080,8090,8084,8093,8096,8098,8105,8106,8107,8113,8111,8116,8117,8121,8120,8127,8125,8146,8157,8166,8166,8171,8176,8189,8194,8197,8228,8218,8224,8229,8232,8242,8237,8252,8250,8255,8263,8267,8280,8295,8291,8299,8309,8305,8319,8313,8318,8321,8320,8325,8326,8342,8343,8356,8353,8368,8375,8375,8373,8378,8393,8390,8403,8405,8408,8424,8438,8438,8440,8446,8465,8463,8476,8479,8491,8508,8497,8500,8504,8513,8525,8522,8529,8540,8544,8548,8547,8561,8569,8569,8571,8574,8579,8597,8596,8593,8608,8604,8604,8609,8608,8613,8612,8613,8620,8622,8621,8624,8628,8627,8634,8632,8637,8654,8646,8649,8655,8657,8656,8667,8677,8675,8698,8691,8690,8693,8705,8708,8717,8720,8719,8719,8724,8723,8728,8727,8730,8736,8738,8737,8744,8750,8751,8766,8763,8772,8786,8784,8782,8783,8790,8792,8805,8807,8808,8825,8821,8828,8825,8832,8834,8843,8841,8849,8848,8850,8850,8857,8861,8859,8864,8863,8868,8867,8872,8877,8875,8886,8884,8892,8899,8897,8899,8905,8909,8907,8918,8920,8918,8919,8922,8930,8941,8935,8936,8943,8947,8945,8956,8958,8956,8957,8960,8968,8975,8973,8973,8974,8981,8985,8983,8994,8996,8994,8995,8998,9006,9013,9011,9011,9012,9019,9023,9021,9032,9034,9032,9037,9046,9060,9054,9054,9065,9067,9065,9084,9076,9081,9088,9095,9109,9103,9103,9106,9112,9116,9126,9127,9125,9126,9139,9144,9158,9154,9155,9161,9165,9163,9174,9178,9181,9179,9192,9197,9211,9205,9205,9208,9208,9209,9212,9212,9219,9217,9218,9221,9227,9233,9240,9254,9252,9250,9253,9261,9272,9270,9273,9282,9280,9282,9286,9285,9292,9292,9296,9301,9301,9319,9313,9313,9318,9319,9319,9322,9328,9326,9333,9349,9347,9363,9357,9365,9372,9386,9380,9380,9383,9389,9393,9391,9410,9408,9406,9409,9415,9421,9438,9429,9432,9434,9436,9436,9437,9440,9440,9455,9459,9457,9465,9480,9472,9481,9490,9491,9489,9492,9494,9496,9508,9525,9527,9526,9527,9531,9530,9537,9535,9542,9542,9548,9546,9559,9580,9568,9575,9591,9595,9604,9604,9604,9606,9606,9607,9610,9610,9617,9615,9622,9624,9637,9637,9641,9649,9671,9659,9660,9669,9682,9691,9684,9691,9707,9719,9722,9721,9720,9723,9729,9733,9747,9756,9763,9765,9764,9780,9781,9784,9796,9802,9799,9813,9807,9814,9822,9824,9824,9828,9827,9834,9832,9837,9850,9844,9847,9853,9857,9867,9864,9864,9869,9868,9873,9874,9884,9906,9894,9903,9906,9904,9913,9914,9923,9927,9932,9939,9949,9943,9944,9945,9946,9947,9950,9950,9951,9958,9956,9967,9975,9988,9984,10003,10012,10004,10013,10032,10028,10034,10047,10044,10069,10065,10082,10081,10080,10082,10084,10086,10092,10090,10095,10100,10114,10111,10136,10130,10134,10138,10142,10146,10148,10153,10159,10160,10167,10172,10177,10174,10183,10190,10196,10192,10203,10215,10210,10220,10228,10223,10232,10235,10247,10243,10251,10256,10257,10278,10284,10291,10303,10297,10301,10302,10306,10317,10320,10321,10326,10346,10356,10361,10379,10378,10374,10385,10393,10400,10413,10407,10417,10429,10442,10441,10448,10450,10456,10473,10474,10478,10477,10481,10480,10483,10483,10486,10488,10490,10492,10494,10494,10497,10499,10505,10507,10510,10511,10519,10522,10520,10527,10525,10532,10534,10537,10536,10549,10556,10559,10562,10576,10577,10574,10581,10579,10582,10582,10601,10599,10607,10603,10610,10614,10612,10615,10615,10632,10639,10633,10636,10638,10640,10650,10660,10664,10666,10674,10694,10701,10703,10699,10710,10710,10713,10732,10729,10727,10734,10742,10740,10746,10752,10763,10763,10771,10781,10779,10794,10793,10799,10812,10813,10822,10829,10838,10844,10843,10850,10859,10862,10865,10875,10874,10879,10877,10892,10886,10889,10891,10893,10897,10914,10914,10917,10933,10933,10938,10941,10951,10958,10957,10964,10969,10974,10981,10977,10982,10985,10994,11016,11012,11019,11015,11028,11031,11042,11040,11057,11069,11072,11067,11072,11083,11084,11092,11100,11099,11112,11112,11118,11123,11122,11124,11126,11132,11136,11134,11139,11142,11149,11156,11160,11165,11168,11176,11180,11199,11197,11194,11205,11203,11221,11213,11224,11230,11242,11235,11248,11251,11264,11265,11266,11271,11283,11297,11292,11311,11302,11327,11324,11321,11328,11334,11337,11343,11350,11362,11363,11360,11371,11379,11376,11376,11379,11381,11387,11385,11392,11404,11414,11417,11421,11426,11423,11432,11433,11436,11451,11448,11469,11469,11465,11468,11470,11476,11484,11486,11494,11490,11493,11495,11513,11511,11527,11529,11539,11541,11546,11545,11545,11550,11559,11557,11559,11561,11565,11572,11573,11578,11582,11588,11587,11595,11608,11609,11618,11621,11624,11626,11637,11640,11641,11654,11656,11661,11676,11678,11674,11677,11679,11703,11704,11705,11711,11726,11724,11728,11728,11738,11748,11748,11768,11766,11767,11784,11780,11787,11804,11809,11825,11821,11833,11835,11840,11849,11854,11866,11863,11870,11877,11895,11889,11904,11897,11900,11902,11908,11922,11929,11927,11942,11951,11944,11953,11960,11976,11967,11974,11994,11992,11999,12005,12001,12012,12010,12012,12028,12033,12044,12043,12051,12067,12074,12067,12078,12090,12097,12095,12108,12114,12113,12120,12117,12134,12147,12138,12145,12157,12157,12163,12170,12186,12185,12184,12198,12198,12208,12209,12219,12222,12229,12247,12249,12244,12267,12261,12276,12281,12290,12284,12307,12303,12309,12314,12331,12326,12341,12334,12343,12350,12364,12372,12379,12373,12394,12403,12415,12422,12431,12424,12433,12444,12442,12455,12461,12464,12473,12470,12484,12478,12485,12491,12504,12537,12538,12537,12536,12544,12553,12548,12559,12571,12570,12568,12585,12594,12589,12601,12618,12626,12624,12636,12644,12646,12643,12646,12648,12662,12658,12672,12674,12681,12693,12692,12702,12709,12716,12723,12724,12730,12727,12736,12749,12752,12748,12751,12753,12767,12773,12784,12785,12788,12808,12808,12804,12817,12816,12829,12831,12834,12840,12843,12859,12861,12871,12865,12882,12875,12886,12902,12913,12910,12926,12922,12927,12945,12950,12952,12961,12964,12973,12980,12995,12998,13001,13014,13013,13021,13027,13037,13031,13030,376,144,195],
+ },
+ 'gateway-gpt-5.4-nano-2000t': {
+ id: 'gateway-gpt-5.4-nano-2000t',
+ boundary: 'gateway',
+ model: 'openai/gpt-5.4-nano',
+ capturedAt: '2026-08-12',
+ eveVersion: '0.33.3',
+ eveCommit: '',
+ events: 1765,
+ spanMs: 19943,
+ totalBytes: 330009,
+ // biome-ignore format: generated capture data
+ offsetsMs: [0,11,20,31,41,56,66,77,86,96,108,128,163,173,183,194,204,214,224,239,247,263,280,291,304,317,339,340,351,358,368,378,393,401,411,427,448,458,473,497,507,518,529,537,548,557,577,590,598,606,620,626,637,648,668,676,687,697,707,723,734,746,754,764,774,784,797,806,817,826,840,847,860,867,876,886,897,912,924,934,949,954,964,974,984,994,1004,1014,1025,1034,1044,1055,1065,1075,1085,1095,1106,1116,1125,1142,1152,1163,1173,1183,1208,1218,1232,1242,1253,1278,1288,1298,1310,1320,1331,1341,1351,1361,1371,1390,1400,1410,1421,1431,1441,1463,1475,1482,1492,1518,1528,1538,1559,1567,1577,1588,1597,1608,1619,1632,1638,1649,1661,1668,1678,1689,1699,1710,1719,1730,1746,1756,1761,1774,1782,1792,1802,1812,1822,1834,1842,1855,1870,1875,1886,1896,1907,1917,1926,1936,1946,1969,1979,1989,1999,2009,2022,2034,2043,2053,2075,2085,2096,2105,2115,2126,2137,2148,2156,2166,2179,2189,2200,2210,2223,2233,2252,2254,2270,2284,2294,2305,2315,2325,2336,2345,2361,2373,2383,2394,2407,2416,2426,2436,2446,2461,2471,2494,2505,2515,2525,2542,2555,2583,2592,2602,2612,2623,2634,2643,2653,2663,2677,2687,2703,2708,2725,2736,2746,2756,2766,2777,2787,2802,2812,2822,2836,2847,2857,2868,2877,2887,2898,2910,2917,2927,2941,2948,2959,2968,2978,2988,2998,3019,3025,3035,3046,3057,3067,3077,3087,3098,3107,3117,3141,3151,3161,3171,3181,3191,3201,3212,3222,3232,3263,3273,3283,3293,3303,3316,3325,3334,3345,3355,3364,3385,3396,3405,3415,3425,3435,3445,3456,3466,3477,3486,3496,3506,3516,3526,3536,3546,3556,3568,3576,3588,3618,3628,3638,3662,3672,3682,3692,3702,3713,3723,3733,3743,3753,3767,3778,3788,3798,3809,3818,3828,3839,3851,3858,3868,3878,3898,3908,3918,3928,3938,3948,3958,3968,3978,3988,3998,4008,4019,4029,4041,4051,4065,4075,4087,4098,4108,4118,4128,4138,4148,4158,4168,4178,4189,4199,4209,4225,4235,4247,4255,4265,4275,4300,4311,4320,4330,4341,4351,4361,4371,4381,4391,4401,4411,4421,4431,4441,4459,4468,4478,4489,4499,4512,4519,4529,4543,4552,4560,4575,4589,4592,4607,4610,4633,4661,4663,4671,4682,4691,4701,4711,4722,4732,4742,4752,4762,4773,4782,4792,4803,4812,4822,4833,4843,4853,4863,4873,4883,4894,4904,4914,4927,4937,4948,4957,4968,4978,4988,4998,5009,5020,5030,5039,5057,5067,5077,5087,5097,5107,5128,5139,5148,5159,5170,5179,5189,5199,5209,5219,5229,5239,5249,5261,5269,5279,5295,5304,5314,5325,5344,5348,5359,5370,5378,5389,5398,5408,5418,5432,5575,5600,5653,5674,5905,5916,5926,5944,6088,6102,6122,6132,6143,6152,6162,6184,6185,6192,6203,6213,6223,6243,6244,6262,6264,6274,6285,6294,6304,6315,6325,6335,6346,6355,6368,6376,6386,6396,6413,6460,6461,6462,6462,6470,6476,6487,6504,6507,6518,6527,6537,6547,6557,6568,6579,6588,6601,6610,6619,6631,6657,6667,6681,6687,6697,6731,6732,6733,6737,6748,6759,6768,6778,6788,6801,6808,6819,6829,6843,6849,6859,6869,6879,6907,6909,7060,7062,7063,7063,7063,7063,7064,7064,7070,7223,7224,7224,7225,7226,7226,7226,7226,7226,7227,7248,7250,7257,7274,7275,7293,7296,7309,7319,7329,7340,7350,7363,7369,7412,7414,7414,7415,7420,7431,7441,7455,7460,7470,7486,7491,7502,7511,7523,7531,7541,7551,7561,7571,7581,7591,7601,7618,7625,7632,7644,7654,7664,7675,7685,7699,7710,7720,7730,7740,7752,7760,7770,7780,7790,7801,7811,7821,7833,7845,7854,7863,7874,7884,7894,7904,7914,7924,7934,7944,7954,7964,7975,7985,7994,8004,8014,8025,8035,8045,8057,8065,8076,8085,8096,8106,8120,8126,8140,8147,8157,8167,8177,8188,8197,8210,8217,8244,8255,8264,8284,8286,8304,8306,8314,8324,8335,8347,8357,8367,8377,8387,8398,8408,8419,8428,8438,8449,8459,8469,8479,8492,8499,8509,8519,8534,8540,8553,8562,8573,8583,8596,8610,8628,8642,8660,8675,8698,8705,8715,8726,8736,8746,8758,8766,8776,8786,8797,8807,8817,8827,8837,8847,8858,8869,8878,8888,8898,8910,8918,8928,8938,8949,8959,8968,8979,8989,8999,9012,9023,9032,9042,9054,9062,9076,9086,9097,9106,9117,9125,9140,9145,9153,9163,9173,9183,9194,9204,9214,9224,9234,9244,9254,9267,9277,9303,9304,9305,9316,9326,9336,9348,9356,9369,9378,9388,9397,9406,9418,9428,9438,9448,9459,9469,9481,9496,9501,9510,9519,9530,9540,9551,9560,9575,9584,9594,9607,9617,9631,9648,9658,9677,9688,9702,9709,9719,9729,9739,9750,9761,9770,9780,9790,9800,9811,9820,9830,9842,9851,9873,9873,9883,9893,9903,9914,9924,9934,9945,9955,9965,9975,9985,9995,10005,10020,10030,10040,10050,10060,10074,10084,10095,10104,10115,10132,10142,10152,10167,10173,10183,10193,10203,10213,10223,10233,10243,10254,10264,10292,10305,10313,10323,10333,10345,10356,10365,10376,10386,10395,10405,10418,10426,10436,10446,10457,10466,10476,10486,10497,10507,10517,10528,10537,10550,10561,10569,10579,10589,10599,10609,10619,10629,10639,10649,10667,10672,10682,10691,10702,10712,10722,10732,10742,10752,10762,10772,10782,10792,10802,10813,10824,10833,10844,10853,10863,10874,10884,10894,10904,10914,10924,10934,10945,10955,10965,10975,10986,10996,11006,11016,11026,11037,11048,11060,11068,11078,11090,11109,11112,11121,11131,11141,11151,11161,11172,11182,11192,11202,11214,11223,11233,11247,11253,11267,11273,11283,11293,11305,11314,11327,11336,11348,11358,11369,11378,11389,11398,11410,11418,11429,11439,11449,11460,11469,11480,11489,11499,11510,11520,11530,11542,11550,11561,11570,11580,11591,11601,11614,11622,11631,11642,11652,11662,11673,11682,11693,11706,11713,11724,11734,11748,11753,11764,11774,11792,11807,11810,11823,11842,11845,11854,11863,11874,11884,11894,11904,11914,11924,11937,11947,11957,11968,11978,11988,12000,12010,12020,12030,12040,12050,12066,12076,12086,12096,12107,12118,12126,12137,12147,12158,12167,12177,12190,12200,12232,12233,12233,12239,12249,12258,12268,12278,12288,12298,12309,12319,12330,12340,12350,12362,12373,12383,12393,12403,12413,12423,12433,12443,12453,12464,12475,12483,12494,12504,12517,12527,12534,12544,12554,12565,12575,12587,12598,12608,12619,12629,12648,12651,12659,12670,12680,12690,12700,12711,12722,12738,12746,12757,12772,12776,12786,12796,12806,12817,12827,12837,12847,12857,12869,12878,12888,12898,12909,12919,12929,12941,12964,12966,12971,12981,12991,13001,13011,13021,13032,13041,13052,13063,13074,13082,13092,13102,13112,13123,13145,13146,13153,13163,13173,13184,13197,13207,13217,13227,13238,13248,13258,13268,13278,13289,13304,13308,13318,13328,13339,13350,13359,13374,13384,13395,13405,13415,13425,13435,13446,13456,13466,13476,13486,13496,13506,13517,13526,13539,13549,13560,13570,13580,13590,13600,13610,13620,13630,13641,13651,13661,13671,13681,13691,13705,13715,13727,13736,13746,13756,13771,13782,13795,13802,13813,13823,13834,13845,13856,13864,13874,13886,13895,13905,13915,13926,13935,13945,13955,13967,13978,13990,14005,14020,14031,14044,14051,14061,14072,14082,14092,14102,14112,14123,14133,14144,14155,14164,14174,14197,14207,14220,14227,14237,14249,14261,14271,14281,14291,14308,14318,14328,14340,14348,14359,14375,14381,14392,14402,14412,14423,14432,14443,14454,14464,14474,14484,14494,14505,14517,14525,14539,14550,14559,14569,14579,14589,14599,14610,14620,14630,14640,14655,14660,14670,14681,14692,14701,14711,14721,14732,14742,14752,14762,14772,14784,14793,14803,14815,14825,14837,14847,14857,14868,14878,14888,14898,14910,14928,14940,14954,14969,14984,14998,15014,15029,15045,15060,15067,15077,15088,15097,15107,15118,15128,15138,15148,15158,15169,15180,15188,15199,15209,15219,15229,15240,15250,15260,15270,15280,15297,15309,15315,15325,15335,15346,15355,15368,15378,15405,15406,15415,15443,15453,15463,15473,15487,15494,15504,15516,15525,15535,15545,15555,15567,15578,15587,15598,15609,15618,15629,15639,15662,15669,15681,15691,15702,15726,15740,15751,15766,15775,15785,15795,15806,15816,15830,15836,15846,15857,15869,15878,15888,15903,15911,15922,15931,15940,15951,15960,15972,15981,15991,16001,16011,16042,16052,16070,16081,16096,16109,16118,16127,16147,16148,16156,16166,16182,16187,16197,16209,16219,16233,16239,16251,16261,16271,16281,16291,16302,16312,16322,16336,16349,16359,16367,16379,16388,16398,16408,16418,16430,16441,16451,16461,16471,16481,16493,16501,16511,16522,16537,16541,16553,16565,16574,16584,16594,16618,16627,16637,16647,16660,16668,16679,16688,16698,16708,16718,16729,16738,16749,16759,16770,16779,16789,16800,16812,16819,16830,16839,16850,16860,16891,16892,16897,16907,16917,16927,16938,16949,16966,16969,16977,16988,16998,17008,17020,17031,17044,17055,17065,17075,17086,17095,17105,17117,17125,17135,17152,17158,17165,17196,17196,17196,17207,17215,17225,17238,17246,17257,17266,17277,17301,17302,17309,17316,17327,17337,17347,17357,17369,17377,17388,17398,17411,17418,17428,17438,17449,17459,17469,17490,17491,17499,17510,17522,17531,17541,17552,17562,17573,17588,17606,17618,17628,17638,17648,17658,17669,17679,17689,17699,17710,17720,17758,17759,17767,17779,17788,17798,17809,17825,17835,17845,17856,17870,17875,17889,17897,17907,17919,17927,17938,17947,17958,17968,17978,17988,17998,18009,18018,18028,18044,18049,18070,18081,18090,18100,18110,18120,18130,18140,18161,18173,18193,18203,18211,18227,18253,18269,18275,18284,18294,18305,18314,18325,18336,18347,18359,18369,18380,18394,18400,18409,18420,18430,18442,18450,18461,18474,18485,18495,18506,18518,18529,18539,18551,18559,18570,18580,18590,18600,18610,18621,18631,18641,18651,18661,18671,18681,18691,18701,18711,18721,18731,18742,18752,18762,18772,18782,18792,18802,18813,18823,18837,18843,18858,18863,18874,18887,18900,18911,18920,18932,18940,18950,18960,18984,18996,19024,19044,19129,19139,19149,19158,19168,19179,19193,19199,19210,19219,19229,19239,19249,19263,19270,19280,19290,19301,19310,19323,19331,19358,19360,19362,19377,19385,19395,19405,19415,19425,19436,19446,19460,19466,19478,19491,19496,19506,19517,19527,19537,19550,19557,19567,19577,19587,19606,19610,19617,19628,19638,19648,19660,19670,19681,19691,19701,19711,19721,19732,19742,19752,19762,19772,19784,19803,19871,19941,19943],
+ // biome-ignore format: generated capture data
+ sizes: [219,212,105,223,211,112,318,104,113,106,110,107,105,536,213,111,106,114,105,111,105,112,108,104,105,103,103,106,323,209,219,103,108,431,215,113,105,109,106,106,109,214,105,220,113,107,108,211,536,106,106,321,110,106,106,211,109,208,416,103,105,218,212,212,105,108,104,215,221,109,209,107,222,103,320,221,107,108,106,430,222,218,107,209,112,219,105,319,105,106,108,319,211,107,105,316,211,111,216,105,315,107,212,212,105,415,106,105,207,104,532,106,106,336,207,208,328,104,206,103,107,217,104,208,210,327,106,105,107,106,323,318,108,215,213,530,107,211,114,316,106,221,104,215,219,212,109,211,210,322,107,209,105,103,105,104,216,213,211,104,326,104,312,106,103,209,103,104,108,103,210,108,315,105,108,325,103,215,213,210,108,429,213,111,103,328,108,213,418,103,105,106,103,422,105,105,211,103,103,105,104,105,106,207,313,210,103,105,208,210,105,106,214,108,209,109,326,104,319,209,218,105,111,113,640,317,216,215,103,106,436,206,103,215,104,104,104,223,215,107,220,106,318,104,215,105,106,321,217,105,218,221,109,210,107,213,112,211,112,218,218,224,211,111,106,322,318,103,108,212,215,104,104,313,105,322,211,110,328,103,213,210,107,328,105,426,215,209,212,104,209,109,216,114,317,104,217,107,319,108,103,209,217,215,107,104,313,103,106,109,215,209,214,221,105,105,103,423,331,105,325,210,212,106,214,103,223,222,217,105,210,214,210,105,105,211,107,218,213,112,109,106,215,328,208,220,105,210,220,108,103,112,106,320,322,105,225,104,208,104,103,105,104,209,106,211,220,105,210,207,105,209,105,210,325,105,106,108,108,317,419,103,215,108,210,104,213,109,105,329,106,212,216,106,108,313,210,106,110,106,215,207,104,209,105,213,105,315,216,105,537,105,214,104,216,219,105,104,113,217,208,317,105,104,107,213,328,103,104,221,103,105,216,105,316,212,315,103,105,108,211,208,214,105,209,112,314,103,105,105,208,318,103,208,103,105,213,106,103,103,208,106,206,105,209,105,106,103,106,106,106,105,313,528,425,103,103,214,222,106,103,103,106,212,106,109,111,105,104,111,109,106,108,106,110,103,220,105,215,216,208,326,111,323,213,220,217,219,219,216,310,210,216,318,218,209,211,106,325,104,210,322,211,208,106,652,212,317,208,207,322,218,104,222,107,317,105,209,216,208,212,218,207,103,432,106,530,429,323,1062,1383,1180,1049,1398,1060,1064,106,212,110,211,112,209,215,218,104,221,106,217,105,214,966,419,107,220,208,103,322,220,105,315,103,320,421,107,329,211,211,319,323,111,213,223,113,223,320,206,103,107,214,219,105,210,108,210,107,217,213,105,217,216,217,105,210,110,213,214,214,108,216,105,217,103,106,110,113,216,221,105,319,103,106,112,105,424,105,105,209,104,313,206,209,104,105,103,105,208,315,104,104,226,212,106,210,110,219,211,208,103,109,107,214,106,217,107,225,106,207,213,108,211,105,218,216,105,328,103,106,211,217,104,217,107,217,103,222,221,103,105,106,108,218,209,104,208,106,209,104,206,315,104,103,209,209,210,103,103,208,208,217,103,103,220,210,106,213,105,214,325,103,213,108,112,106,103,112,105,104,105,104,103,210,215,109,212,111,319,103,107,211,116,213,110,311,104,111,321,104,103,209,209,106,209,107,218,212,105,333,105,106,215,104,220,209,106,212,211,106,216,216,113,113,209,113,106,108,212,105,107,217,217,103,218,108,105,216,212,106,212,107,213,214,105,213,325,109,104,206,103,110,216,214,107,220,214,103,223,213,106,213,105,217,313,105,320,104,104,105,111,111,108,104,105,103,106,103,110,311,211,108,108,106,210,324,108,315,103,105,424,104,105,105,424,104,104,105,209,104,209,309,104,214,214,210,104,103,209,103,420,104,110,423,104,105,105,103,112,106,103,419,105,216,209,106,324,210,105,106,111,318,210,222,107,110,212,225,209,103,219,215,105,209,216,111,215,108,213,219,111,223,214,216,320,103,103,209,103,209,110,221,109,325,104,310,104,212,111,212,105,107,318,106,325,103,210,107,216,209,217,104,216,109,213,208,106,106,209,213,216,106,315,105,108,212,216,220,103,209,105,215,216,107,213,212,206,206,103,105,105,104,106,105,310,108,210,212,103,105,209,211,108,209,217,220,110,212,208,103,225,107,217,208,209,213,111,211,212,105,214,210,104,218,107,104,319,209,211,216,109,217,217,109,213,104,214,107,103,222,113,221,216,331,213,206,103,111,220,208,216,214,215,213,106,315,103,210,418,105,219,104,209,209,211,214,103,211,105,330,103,219,216,314,209,104,104,103,209,103,104,522,104,208,209,105,217,207,210,109,210,209,217,105,318,217,103,104,217,216,219,106,207,210,110,105,213,209,104,218,316,103,214,210,109,215,215,103,314,209,103,215,209,219,104,211,209,107,312,209,103,315,207,207,103,103,106,107,316,103,326,213,221,208,216,108,424,104,103,209,104,315,103,207,208,206,104,206,104,206,106,213,224,109,215,216,106,215,213,217,106,210,316,103,225,209,210,210,110,208,219,209,213,219,106,222,216,218,217,212,113,323,103,219,209,213,103,314,211,103,211,106,208,312,104,208,310,104,310,103,104,213,210,219,106,224,211,209,218,109,211,330,106,218,216,105,211,103,223,315,106,217,221,216,106,322,103,214,218,106,419,104,103,209,104,105,210,103,103,213,316,103,107,218,216,107,216,103,218,209,107,214,211,207,212,211,106,221,108,104,103,109,107,106,222,109,318,103,319,104,208,311,105,211,219,104,521,104,105,103,210,311,103,109,107,111,104,215,207,208,108,311,211,103,415,105,103,221,111,209,104,208,415,103,107,215,107,325,103,212,214,107,213,213,214,215,107,209,105,104,315,325,104,218,215,104,112,109,218,111,214,220,216,106,107,108,639,103,108,212,222,103,208,104,106,104,104,106,109,107,107,106,112,107,107,105,217,109,325,110,215,219,219,105,217,220,112,109,109,216,105,312,104,214,209,112,320,104,415,105,213,113,110,112,105,108,107,105,213,103,430,104,320,105,103,217,316,108,213,221,210,103,212,103,105,420,104,109,222,212,106,103,105,103,104,109,114,105,323,104,110,220,216,215,104,215,216,215,107,321,103,212,112,326,103,208,104,321,224,315,105,208,105,106,103,103,105,310,105,105,209,311,207,208,104,313,210,103,108,313,210,103,105,106,109,212,217,213,313,103,105,109,211,108,217,218,108,321,107,103,104,324,107,219,111,214,207,209,215,221,109,213,221,313,104,107,416,220,213,209,112,107,213,104,207,207,213,214,212,105,221,217,211,211,113,107,318,207,104,216,319,309,105,103,213,112,104,104,105,221,107,211,315,309,105,311,104,104,108,328,214,218,212,103,215,111,216,211,219,218,103,208,213,108,217,215,112,107,216,216,109,210,109,217,213,107,212,213,207,104,211,216,107,329,103,217,105,322,106,214,105,218,103,108,530,105,103,211,103,103,107,210,105,108,211,209,107,215,211,207,211,103,316,105,108,211,316,103,105,107,211,103,314,105,318,103,106,215,313,103,208,109,219,220,211,104,218,108,319,110,219,218,107,216,223,318,104,107,108,106,113,104,103,104,104,112,105,226,104,214,113,212,221,115,107,108,113,320,110,209,329,629,332,1166,422,216,321,105,208,106,312,105,322,106,104,215,105,321,109,326,106,208,216,219,208,105,107,218,208,211,212,105,214,106,327,110,211,104,312,211,111,318,104,212,212,111,211,114,107,107,105,104,212,215,113,103,107,110,105,113,217,319,218,105,207,207,214,222,107,112,218,327,210,434,208,109,218,207,105,103,208,104,107,106,215,212,316,108,649,545,646,416,755,427,219,109,319,216,109,211,213,207,213,110,104,329,104,208,209,104,417,103,213,215,211,106,213,209,208,218,103,216,107,209,103,210,1537,6],
+ },
+};
diff --git a/workbench/example/workflows/97_bench_rtt.ts b/workbench/example/workflows/97_bench_rtt.ts
new file mode 100644
index 0000000000..abcf8c490e
--- /dev/null
+++ b/workbench/example/workflows/97_bench_rtt.ts
@@ -0,0 +1,390 @@
+// Pure bucketing + aggregation helpers for the chunk round-trip-time (CRTT)
+// benchmark scenario. The workflow half lives in 97_bench.ts
+// (benchCrttWorkflow) and the runner half in
+// packages/core/e2e/benchmark.test.ts.
+//
+// This module is deliberately dependency-free so the same code runs in three
+// places: the reader step aggregates per-chunk RTT samples on the deployment
+// (keeping the workflow return value small — bucketed summaries, not hundreds
+// of raw samples), the benchmark runner merges the per-iteration summaries
+// into one row per bucket, and the unit tests
+// (packages/core/src/bench-chunk-rtt-stats.test.ts) exercise both directly.
+
+/**
+ * Summary of one bucket's RTT samples (all values in ms, rounded to 0.1ms).
+ * Computed inside the reader step per iteration (exact percentiles over that
+ * iteration's samples), then merged across iterations by
+ * {@link mergeRttSummaries}.
+ */
+export interface BenchRttSummary {
+ /** Number of samples aggregated into this summary. */
+ count: number;
+ /** Fastest sample (min). */
+ best: number;
+ /** Mean — the exit criteria's headline "average per-chunk RTT". */
+ avg: number;
+ p50: number;
+ p75: number;
+ p90: number;
+ p99: number;
+ /** Fixed-bin histogram of the samples (see {@link RTT_HIST_EDGES_MS}):
+ * `hist[i]` counts samples in `[edges[i-1], edges[i])`, with `hist[0]`
+ * below the first edge and the last entry at/above the last edge. Because
+ * the edges are a shared constant, histograms merge exactly — across
+ * iterations and across benchmark runs — unlike the percentile fields. */
+ hist: number[];
+}
+
+// Histogram bin edges (ms), a 1-2-5 log series. Log-scale bins keep
+// resolution at both ends of the plausible range — a warm in-region
+// write->read can be single-digit ms while a stalled delivery is over a
+// second — and fixed shared edges are what make cross-run histogram diffs
+// exact (adaptive widths, like the STSO section's, cannot be re-binned once
+// the raw samples have been left behind on the deployment).
+export const RTT_HIST_EDGES_MS = [
+ 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000,
+];
+
+/** Buckets samples into the fixed {@link RTT_HIST_EDGES_MS} bins. Returns
+ * `edges.length + 1` counts (last = at/above the final edge). */
+export function histogramRttSamples(samples: number[]): number[] {
+ const counts = new Array(RTT_HIST_EDGES_MS.length + 1).fill(0);
+ for (const v of samples) {
+ let bin = 0;
+ while (bin < RTT_HIST_EDGES_MS.length && v >= RTT_HIST_EDGES_MS[bin]) {
+ bin++;
+ }
+ counts[bin]++;
+ }
+ return counts;
+}
+
+// Chunk-index buckets. Each boundary is tied to a mechanism, not a progress
+// range:
+// - 'seq 0': the stream-open write (stream creation / cold write path). Also
+// a cross-check against the SL scenario, which times the same first-chunk
+// propagation.
+// - 'seq 1-20': warmup — the first ~200ms at the modeled 100 chunks/s, where
+// connections, buffers, and flush cycles are still settling.
+// - 'seq 21+': steady state, kept as ONE bucket so its large n gives stable
+// tail percentiles (splitting it further just compares noise floors of
+// unequal sample sizes — iteration-level stalls land in whichever range
+// they land in).
+// Latency *drift* across the stream (cumulative log/buffer growth) is a
+// trend, which fixed buckets detect badly; that is the progress profile's
+// job (see {@link progressProfile}).
+export const RTT_INDEX_BUCKETS = ['seq 0', 'seq 1-20', 'seq 21+'] as const;
+export type RttIndexBucket = (typeof RTT_INDEX_BUCKETS)[number];
+
+export function rttIndexBucket(seq: number): RttIndexBucket {
+ if (seq <= 0) return 'seq 0';
+ if (seq <= 20) return 'seq 1-20';
+ return 'seq 21+';
+}
+
+// Number of equal fractions of the stream in the progress profile. Ten keeps
+// the profile line compact while still localizing a drift or a slow phase.
+export const RTT_PROGRESS_BINS = 10;
+
+/** A binned mean-RTT profile: `totalMs[i]`/`counts[i]` is the mean RTT of
+ * bin i. Used for both the stream-progress profile (bin = tenth of the
+ * stream) and the chunk-size profile (bin = log size range). Sums and counts
+ * merge exactly across iterations and runs. */
+export interface BenchRttMeanProfile {
+ counts: number[];
+ totalMs: number[];
+}
+
+/** Builds the progress profile from per-seq RTT samples (`rttBySeq[seq]` =
+ * that chunk's RTT; sparse entries are skipped defensively). Fraction-based
+ * (not absolute seq), so profiles are comparable across chunk counts. The
+ * trend this surfaces — does per-chunk RTT rise as the stream grows? — is
+ * what fixed index buckets cannot answer without arbitrary boundaries. */
+export function progressProfile(
+ rttBySeq: readonly (number | undefined)[]
+): BenchRttMeanProfile {
+ const counts = new Array(RTT_PROGRESS_BINS).fill(0);
+ const totalMs = new Array(RTT_PROGRESS_BINS).fill(0);
+ const n = rttBySeq.length;
+ for (let seq = 0; seq < n; seq++) {
+ const rtt = rttBySeq[seq];
+ if (typeof rtt !== 'number') continue;
+ const bin = Math.min(
+ RTT_PROGRESS_BINS - 1,
+ Math.floor((seq * RTT_PROGRESS_BINS) / n)
+ );
+ counts[bin]++;
+ totalMs[bin] += rtt;
+ }
+ return { counts, totalMs };
+}
+
+/** Tail summary of a delay-style sample set where the MAX is the headline
+ * (one bad event among hundreds vanishes into pooled percentiles but is, by
+ * construction, the max). Used for write slip (producer-side lateness vs the
+ * open-loop schedule) and for positive CDV (delivery clumps/stalls). */
+export interface BenchDelayTail {
+ count: number;
+ avgMs: number;
+ p99Ms: number;
+ maxMs: number;
+}
+
+/** Summarizes delay samples into a {@link BenchDelayTail}. */
+export function summarizeDelayTail(
+ samples: number[]
+): BenchDelayTail | undefined {
+ if (samples.length === 0) return undefined;
+ const sorted = [...samples].sort((a, b) => a - b);
+ return {
+ count: sorted.length,
+ avgMs: round(sorted.reduce((sum, v) => sum + v, 0) / sorted.length),
+ p99Ms: round(percentile(sorted, 99)),
+ maxMs: round(sorted[sorted.length - 1]),
+ };
+}
+
+/** Sustained throughput over the steady window of a run: the first and last
+ * `trimFraction` of points (by index) are dropped so warmup (first-delivery
+ * setup) and drain (final flush) don't flatter or damn the sustained rate. */
+export interface BenchSteadyRate {
+ chunksPerSec: number;
+ kibPerSec: number;
+ /** Points inside the steady window. */
+ windowChunks: number;
+ /** Wall span of the steady window (ms). */
+ windowMs: number;
+}
+
+/** Computes the steady-window rate from per-chunk (timestamp, bytes) points
+ * in stream order. Returns undefined when the window is too small to define
+ * a rate (fewer than 2 points or zero span). */
+export function steadyRate(
+ points: readonly { atMs: number; bytes: number }[],
+ trimFraction = 0.1
+): BenchSteadyRate | undefined {
+ const trim = Math.floor(points.length * trimFraction);
+ const window = points.slice(trim, points.length - trim);
+ if (window.length < 2) return undefined;
+ const spanMs = window[window.length - 1].atMs - window[0].atMs;
+ if (spanMs <= 0) return undefined;
+ // Both rates count events over the window's intervals: n points span n-1
+ // gaps, and the first point's bytes "arrived" before the window's clock
+ // started — counting them would inflate a perfectly steady stream's
+ // byte rate by 1/(n-1).
+ const bytes = window.slice(1).reduce((sum, p) => sum + p.bytes, 0);
+ const round = (v: number) => Math.round(v * 10) / 10;
+ return {
+ chunksPerSec: round(((window.length - 1) * 1000) / spanMs),
+ kibPerSec: round((bytes * 1000) / spanMs / 1024),
+ windowChunks: window.length,
+ windowMs: spanMs,
+ };
+}
+
+/** One received chunk's RAW timestamps, in arrival order. CDV must be
+ * computed from unclamped values: clamping breaks the telescoping identity
+ * and hides the negative (catch-up) half of every delivery clump. */
+export interface CdvArrival {
+ seq: number;
+ writtenAt: number;
+ readAt: number;
+}
+
+export interface BenchCdvComputation {
+ /** Signed cdv per seq-adjacent arrival pair, in arrival order. */
+ cdvMs: number[];
+ /** Positive cdv indexed by the later chunk's seq — progressProfile input
+ * (length is padded to max seq + 1 so fraction bins line up). */
+ positiveBySeq: (number | undefined)[];
+ duplicateSeqs: number;
+ reorderedArrivals: number;
+ /** Adjacent arrivals skipped because their seqs weren't consecutive. */
+ skippedPairs: number;
+}
+
+/**
+ * Chunk delay variation (delivery jitter): for seq-adjacent chunks received
+ * back to back, cdv_i = (readAt_i - readAt_{i-1}) - (writtenAt_i -
+ * writtenAt_{i-1}) = CTT_i - CTT_{i-1}. Each gap subtracts same-clock
+ * stamps, so CDV is skew-free — measurable in production where cross-clock
+ * CTT is not. Signed: clumped delivery of a 10ms-paced stream reads
+ * (-10, -10, +20); the sum telescopes, so report the positive tail, not
+ * means. Writer pauses self-exclude (both gaps grow equally). Pairs form
+ * only for chunks adjacent in BOTH arrival order and seq (pairing
+ * seq-sorted samples under reordering would manufacture phantom cdv);
+ * duplicates/reorders/holes are counted for the caller to treat as
+ * integrity failures.
+ */
+export function computeCdv(
+ arrivals: readonly CdvArrival[]
+): BenchCdvComputation {
+ const seen = new Set();
+ const cdvMs: number[] = [];
+ const positiveBySeq: (number | undefined)[] = [];
+ let duplicateSeqs = 0;
+ let reorderedArrivals = 0;
+ let skippedPairs = 0;
+ let maxSeq = -1;
+ for (let i = 0; i < arrivals.length; i++) {
+ const chunk = arrivals[i];
+ if (seen.has(chunk.seq)) duplicateSeqs++;
+ seen.add(chunk.seq);
+ if (chunk.seq > maxSeq) maxSeq = chunk.seq;
+ if (i === 0) continue; // the first arrival anchors; it has no pair
+ const prev = arrivals[i - 1];
+ if (chunk.seq < prev.seq) reorderedArrivals++;
+ if (chunk.seq !== prev.seq + 1) {
+ skippedPairs++;
+ continue;
+ }
+ const cdv = chunk.readAt - prev.readAt - (chunk.writtenAt - prev.writtenAt);
+ cdvMs.push(cdv);
+ if (cdv > 0) positiveBySeq[chunk.seq] = cdv;
+ }
+ positiveBySeq.length = Math.max(positiveBySeq.length, maxSeq + 1);
+ return {
+ cdvMs,
+ positiveBySeq,
+ duplicateSeqs,
+ reorderedArrivals,
+ skippedPairs,
+ };
+}
+
+/** Merges mean profiles by summation — exact, like the histograms. */
+export function mergeMeanProfiles(
+ profiles: readonly (BenchRttMeanProfile | undefined)[]
+): BenchRttMeanProfile | undefined {
+ const present = profiles.filter((p): p is BenchRttMeanProfile => p != null);
+ if (present.length === 0) return undefined;
+ const bins = Math.max(...present.map((p) => p.counts.length));
+ const counts = new Array(bins).fill(0);
+ const totalMs = new Array(bins).fill(0);
+ for (const p of present) {
+ for (let i = 0; i < bins; i++) {
+ counts[i] += p.counts[i] ?? 0;
+ totalMs[i] += p.totalMs[i] ?? 0;
+ }
+ }
+ return { counts, totalMs };
+}
+
+// Chunk-size profile bins (approximate serialized bytes, doubling edges).
+// Bin i covers [edges[i-1], edges[i]), bin 0 everything below 256B, and the
+// last bin everything at/above 8KB. The size-sweep scenario's pad rotation
+// (see CRTT_SWEEP_PAD_LENGTHS in 97_bench.ts) puts one padded size in each
+// bin, so the mean-RTT-per-bin profile is a size→latency curve: flat means
+// chunk size doesn't matter, a knee localizes where it starts to.
+export const RTT_SIZE_BIN_EDGES_BYTES = [256, 512, 1024, 2048, 4096, 8192];
+
+/** Bin index into {@link RTT_SIZE_BIN_EDGES_BYTES} for a serialized size. */
+export function rttSizeBin(serializedBytes: number): number {
+ let bin = 0;
+ while (
+ bin < RTT_SIZE_BIN_EDGES_BYTES.length &&
+ serializedBytes >= RTT_SIZE_BIN_EDGES_BYTES[bin]
+ ) {
+ bin++;
+ }
+ return bin;
+}
+
+/** Builds the chunk-size profile from (serialized bytes, RTT) samples. */
+export function sizeProfile(
+ samples: readonly { bytes: number; rttMs: number }[]
+): BenchRttMeanProfile {
+ const bins = RTT_SIZE_BIN_EDGES_BYTES.length + 1;
+ const counts = new Array(bins).fill(0);
+ const totalMs = new Array(bins).fill(0);
+ for (const { bytes, rttMs } of samples) {
+ const bin = rttSizeBin(bytes);
+ counts[bin]++;
+ totalMs[bin] += rttMs;
+ }
+ return { counts, totalMs };
+}
+
+// Same percentile convention as the benchmark runner's computeStats
+// (nearest-rank via ceil), so a CRTT p90 means the same thing as an SO p90.
+function percentile(sortedAscending: number[], q: number): number {
+ return sortedAscending[
+ Math.min(
+ sortedAscending.length - 1,
+ Math.ceil((q / 100) * sortedAscending.length) - 1
+ )
+ ];
+}
+
+const round = (v: number) => Math.round(v * 10) / 10;
+
+/** Exact summary of one iteration's samples for a bucket; undefined when the
+ * bucket received no samples (so the caller can just skip it). */
+export function summarizeRttSamples(
+ samples: number[]
+): BenchRttSummary | undefined {
+ if (samples.length === 0) return undefined;
+ const sorted = [...samples].sort((a, b) => a - b);
+ return {
+ count: sorted.length,
+ best: round(sorted[0]),
+ avg: round(sorted.reduce((sum, v) => sum + v, 0) / sorted.length),
+ hist: histogramRttSamples(sorted),
+ p50: round(percentile(sorted, 50)),
+ p75: round(percentile(sorted, 75)),
+ p90: round(percentile(sorted, 90)),
+ p99: round(percentile(sorted, 99)),
+ };
+}
+
+/**
+ * Merges per-iteration bucket summaries. count/best/avg (count-weighted) and
+ * hist (elementwise over shared fixed bins) are exact; p50-p99 are
+ * percentile-of-percentiles (raw samples never leave the reader step) —
+ * exact only for single-sample iterations (e.g. seq 0), an approximation
+ * for headline rows. Good enough for trend tracking; the histogram is the
+ * exact pooled view.
+ */
+export function mergeRttSummaries(
+ summaries: readonly (BenchRttSummary | undefined)[]
+): BenchRttSummary | undefined {
+ const present = summaries.filter((s): s is BenchRttSummary => s != null);
+ if (present.length === 0) return undefined;
+ const count = present.reduce((sum, s) => sum + s.count, 0);
+ const mergedPercentile = (q: number, values: number[]) =>
+ round(
+ percentile(
+ [...values].sort((a, b) => a - b),
+ q
+ )
+ );
+ const histLength = Math.max(...present.map((s) => s.hist?.length ?? 0));
+ const hist = new Array(histLength).fill(0);
+ for (const s of present) {
+ (s.hist ?? []).forEach((c, i) => {
+ hist[i] += c;
+ });
+ }
+ return {
+ count,
+ best: round(Math.min(...present.map((s) => s.best))),
+ avg: round(present.reduce((sum, s) => sum + s.avg * s.count, 0) / count),
+ hist,
+ p50: mergedPercentile(
+ 50,
+ present.map((s) => s.p50)
+ ),
+ p75: mergedPercentile(
+ 75,
+ present.map((s) => s.p75)
+ ),
+ p90: mergedPercentile(
+ 90,
+ present.map((s) => s.p90)
+ ),
+ p99: mergedPercentile(
+ 99,
+ present.map((s) => s.p99)
+ ),
+ };
+}
diff --git a/workbench/nextjs-turbopack/workflows/97_bench_cadence.ts b/workbench/nextjs-turbopack/workflows/97_bench_cadence.ts
new file mode 120000
index 0000000000..4c9bfa4d0e
--- /dev/null
+++ b/workbench/nextjs-turbopack/workflows/97_bench_cadence.ts
@@ -0,0 +1 @@
+../../example/workflows/97_bench_cadence.ts
\ No newline at end of file
diff --git a/workbench/nextjs-turbopack/workflows/97_bench_rtt.ts b/workbench/nextjs-turbopack/workflows/97_bench_rtt.ts
new file mode 120000
index 0000000000..2463f17ab7
--- /dev/null
+++ b/workbench/nextjs-turbopack/workflows/97_bench_rtt.ts
@@ -0,0 +1 @@
+../../example/workflows/97_bench_rtt.ts
\ No newline at end of file
diff --git a/workbench/nitro-v3/workflows/97_bench_cadence.ts b/workbench/nitro-v3/workflows/97_bench_cadence.ts
new file mode 120000
index 0000000000..4c9bfa4d0e
--- /dev/null
+++ b/workbench/nitro-v3/workflows/97_bench_cadence.ts
@@ -0,0 +1 @@
+../../example/workflows/97_bench_cadence.ts
\ No newline at end of file
diff --git a/workbench/nitro-v3/workflows/97_bench_rtt.ts b/workbench/nitro-v3/workflows/97_bench_rtt.ts
new file mode 120000
index 0000000000..2463f17ab7
--- /dev/null
+++ b/workbench/nitro-v3/workflows/97_bench_rtt.ts
@@ -0,0 +1 @@
+../../example/workflows/97_bench_rtt.ts
\ No newline at end of file