From afe3d789fb0ddbbaccfc7dc460b5a574b4908a60 Mon Sep 17 00:00:00 2001 From: "vercel[bot]" <35613825+vercel[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:00:39 +0000 Subject: [PATCH 1/4] [benchmarks] Link the run id and Datadog trace under the STSO histograms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The STSO distribution section added in #3213 shows the shape of the sequential-steps run but not which run produced it, so investigating an odd-looking bucket meant hunting for the run by deployment id and time window. Capture the identity alongside the samples (the mechanism prototyped on the WIP variance branch, #3107): `/api/bench` returns the trace id of the span @vercel/otel opened for the trigger request, the runner threads it through the sequential iteration and records `sequentialRuns` in the result file, and the renderer prints one line under the histograms with the run id + Datadog trace link for this run and for the `main` run it is diffed against. Every part is optional — a deployment predating the route change yields a bare run id, and a `main` baseline predating this yields only this run's side — so the section degrades instead of breaking on mixed-vintage artifacts. Co-Authored-By: Claude Opus 5 (1M context) Co-Authored-By: Shalabh Chaturvedi <7066873+shalabhc@users.noreply.github.com> Co-Authored-By: shalabhc --- .changeset/bench-histogram-run-trace-links.md | 4 + .github/scripts/render-benchmark-comment.mjs | 53 +++++++++++- .../scripts/render-benchmark-comment.test.js | 81 ++++++++++++++++++- packages/core/e2e/benchmark.test.ts | 34 +++++++- .../nextjs-turbopack/app/api/bench/route.ts | 9 ++- 5 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 .changeset/bench-histogram-run-trace-links.md diff --git a/.changeset/bench-histogram-run-trace-links.md b/.changeset/bench-histogram-run-trace-links.md new file mode 100644 index 0000000000..fccdb09937 --- /dev/null +++ b/.changeset/bench-histogram-run-trace-links.md @@ -0,0 +1,4 @@ +--- +--- + +Record the run id and Datadog trace id of each sequential-steps benchmark iteration and link them under the STSO histograms in the benchmark PR comment, so a suspicious distribution can be opened in APM directly. diff --git a/.github/scripts/render-benchmark-comment.mjs b/.github/scripts/render-benchmark-comment.mjs index a601422ae0..9d975086ac 100644 --- a/.github/scripts/render-benchmark-comment.mjs +++ b/.github/scripts/render-benchmark-comment.mjs @@ -217,8 +217,14 @@ export function annotateWithBaseline(results, baseline) { const methodology = (result) => result.methodologyVersion ?? 'legacy'; const keyFor = (result, row) => `${methodology(result)}/${result.backend}/${result.app}/${row.metric}/${row.scenario}`; + // Same key minus the metric row — the sequential-run identities annotated + // below live on the result, not on an individual metric row. + const resultKeyFor = (result) => + `${methodology(result)}/${result.backend}/${result.app}`; const baselineRows = new Map(); + const baselineResults = new Map(); for (const result of baseline) { + baselineResults.set(resultKeyFor(result), result); for (const row of result.metrics ?? []) { baselineRows.set(keyFor(result, row), row); } @@ -237,10 +243,18 @@ export function annotateWithBaseline(results, baseline) { if (Array.isArray(base.raw)) annotated.baselineRaw = base.raw; return annotated; }; - return results.map((result) => ({ - ...result, - metrics: (result.metrics ?? []).map((row) => annotate(result, row)), - })); + return results.map((result) => { + // The baseline's sequential runs (when it recorded them) let the histogram + // section link both sides of the diff, not just this run's. + const baseResult = baselineResults.get(resultKeyFor(result)); + return { + ...result, + metrics: (result.metrics ?? []).map((row) => annotate(result, row)), + ...(Array.isArray(baseResult?.sequentialRuns) + ? { baselineSequentialRuns: baseResult.sequentialRuns } + : {}), + }; + }); } // ============================================================================ @@ -451,6 +465,36 @@ function renderStsoRowDiff(row) { return lines.join('\n'); } +// Datadog APM permalink for a trace id. The benchmark deployment exports its +// OTel spans to Datadog, and `/api/bench` returns the trace id of the request +// that started each run (see packages/core/e2e/benchmark.test.ts). +const DATADOG_TRACE_URL = 'https://app.datadoghq.com/apm/trace/'; + +/** + * Renders the identity of the sequential-steps run(s) behind the histograms + * above — run id plus a Datadog trace link for each, for this run and for the + * `main` run it is diffed against. Both histograms (inline and queue-hop) come + * out of the same iteration, so this is one line for the whole section rather + * than a repeat under each chart. + * + * Everything is optional: a deployment built before `/api/bench` returned a + * trace id yields a bare run id, and a `main` baseline from before this landed + * has no runs to name at all, in which case only this run's side is shown. + */ +function renderSequentialRunLinks(result) { + const current = result.sequentialRuns ?? []; + const baseline = result.baselineSequentialRuns ?? []; + if (current.length === 0 && baseline.length === 0) return []; + const formatRun = (run) => + `\`${run.runId}\`${run.traceId ? ` ([trace](${DATADOG_TRACE_URL}${run.traceId}))` : ''}`; + const side = (label, runs) => + runs.length > 0 ? `${label}: ${runs.map(formatRun).join(', ')}` : undefined; + const sides = [side('this run', current), side('`main`', baseline)].filter( + Boolean + ); + return ['', `Runs behind these histograms — ${sides.join(' · ')}`]; +} + /** * Renders a per-scenario histogram diff (bucketed step counts) and a * cumulative-time diff (sum of all STSO samples) against `main`, for every @@ -475,6 +519,7 @@ function renderStsoDiffSection(result) { `📈 STSO distribution${anyBaseline ? ' vs main' : ''} (inline / queue-hop histograms)`, '', ...rows.map(renderStsoRowDiff), + ...renderSequentialRunLinks(result), '', '', ].join('\n'); diff --git a/.github/scripts/render-benchmark-comment.test.js b/.github/scripts/render-benchmark-comment.test.js index 2782be6d42..ff763ea1ba 100644 --- a/.github/scripts/render-benchmark-comment.test.js +++ b/.github/scripts/render-benchmark-comment.test.js @@ -461,7 +461,7 @@ test('CLI fails when completed with no results', async () => { /** A sequential-steps result whose STSO rows carry raw samples, as the * benchmark runner now records them (every gap, not a sampled window). */ -function sequentialResult({ inline, queueHop }) { +function sequentialResult({ inline, queueHop, sequentialRuns }) { const stsoRow = (scenario, raw) => ({ metric: 'stso', scenario, @@ -482,6 +482,7 @@ function sequentialResult({ inline, queueHop }) { stsoRow('1020 steps (inline)', inline), stsoRow('1020 steps (queue-hop)', queueHop), ], + ...(sequentialRuns ? { sequentialRuns } : {}), }); } @@ -599,6 +600,84 @@ test('strips raw samples from the embedded history data block', async () => { assert.doesNotMatch(rerendered, /STSO distribution/); }); +test('links the run id and Datadog trace under the histograms', async () => { + const { renderComment } = await loadModule(); + const body = renderComment({ + status: 'completed', + results: [ + sequentialResult({ + inline: [160, 360], + queueHop: [2100], + sequentialRuns: [{ runId: 'run_this', traceId: 'abc123' }], + }), + ], + baseline: [ + sequentialResult({ + inline: [160, 360], + queueHop: [2100], + sequentialRuns: [{ runId: 'run_main', traceId: 'def456' }], + }), + ], + history: [], + commit: 'abcdef1234567890', + }); + + // One line for the whole section — both histograms come from the same run. + assert.match( + body, + /Runs behind these histograms — this run: `run_this` \(\[trace\]\(https:\/\/app\.datadoghq\.com\/apm\/trace\/abc123\)\) · `main`: `run_main` \(\[trace\]\(https:\/\/app\.datadoghq\.com\/apm\/trace\/def456\)\)<\/sub>/ + ); + // ...and it sits inside the collapsed distribution section, below the charts. + const section = body.slice( + body.indexOf('📈 STSO distribution'), + body.indexOf('', body.indexOf('📈 STSO distribution')) + ); + assert.ok( + section.indexOf('Runs behind these histograms') > + section.lastIndexOf('```'), + 'run links should render after the last bar chart' + ); +}); + +test('degrades to bare run ids when no trace or baseline runs exist', async () => { + const { renderComment } = await loadModule(); + const body = renderComment({ + status: 'completed', + results: [ + sequentialResult({ + inline: [160, 360], + queueHop: [2100], + // A deployment built before /api/bench reported a trace id. + sequentialRuns: [{ runId: 'run_this' }], + }), + ], + // A `main` baseline from before the runner recorded run identities. + baseline: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], + history: [], + commit: 'abcdef1234567890', + }); + + assert.match( + body, + /Runs behind these histograms — this run: `run_this`<\/sub>/ + ); + assert.doesNotMatch(body, /datadoghq/); +}); + +test('omits the run links entirely when no run identities were recorded', async () => { + const { renderComment } = await loadModule(); + const body = renderComment({ + status: 'completed', + results: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], + baseline: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], + history: [], + commit: 'abcdef1234567890', + }); + + assert.match(body, /📈 STSO distribution vs main/); + assert.doesNotMatch(body, /Runs behind these histograms/); +}); + test('buckets negative STSO gaps separately from the slow tail', async () => { const { renderComment } = await loadModule(); // Consecutive step timestamps come from different step bodies, so a gap can diff --git a/packages/core/e2e/benchmark.test.ts b/packages/core/e2e/benchmark.test.ts index 312dae14b2..3da53354e0 100644 --- a/packages/core/e2e/benchmark.test.ts +++ b/packages/core/e2e/benchmark.test.ts @@ -193,6 +193,9 @@ interface StreamIterationResult { interface SequentialIterationResult { runId: string; + /** Datadog trace id for the `/api/bench` request that started this run, when + * the deployment's route reports one (older deployments won't). */ + traceId?: string; /** STSO gaps preceding an 'inline' step (same warm process as the step * before it) — the framework's pure step-to-step overhead. */ stsoInlineMs: number[]; @@ -221,6 +224,8 @@ interface BenchTriggerResponse { runId: string; /** Date.now() stamped in the route immediately before start(). */ clientStart: number; + /** Datadog trace id for this request, when the route reports one. */ + traceId?: string; } function withTimeout( @@ -272,7 +277,13 @@ async function triggerBenchRun( `bench trigger for ${workflowFn} returned malformed body: ${JSON.stringify(data)?.slice(0, 200)}` ); } - return { runId: data.runId, clientStart: data.clientStart }; + return { + runId: data.runId, + clientStart: data.clientStart, + // Optional: a deployment built before the route reported it simply omits + // the trace link from the rendered comment. + traceId: typeof data.traceId === 'string' ? data.traceId : undefined, + }; } /** Poll a run's return value to completion (the handle polls internally). */ @@ -340,7 +351,7 @@ async function runStreamIteration( async function runSequentialIteration( stepCount: number ): Promise { - const { runId, clientStart } = await triggerBenchRun( + const { runId, clientStart, traceId } = await triggerBenchRun( 'benchSequentialStepsWorkflow', [stepCount] ); @@ -366,6 +377,7 @@ async function runSequentialIteration( return { runId, + traceId, stsoInlineMs, stsoQueueHopMs, woMs: workflowOverheadMs(clientStart, steps), @@ -561,6 +573,17 @@ interface MetricRow extends MetricStats { const metricRows: MetricRow[] = []; +/** Identity of one sequential-steps iteration. */ +interface SequentialRunRef { + runId: string; + traceId?: string; +} + +/** The runs behind the STSO histograms, recorded so the PR comment can link + * straight to each run and its Datadog trace — which is where a + * suspicious-looking distribution has to be investigated. */ +const sequentialRuns: SequentialRunRef[] = []; + function recordMetric( metric: string, scenario: string, @@ -777,6 +800,9 @@ describe('workflow benchmarks', () => { extraAttempts: Math.max(2, Math.ceil(SEQUENTIAL_ITERATIONS * 0.5)), } ); + sequentialRuns.push( + ...results.map((r) => ({ runId: r.runId, traceId: r.traceId })) + ); // Report STSO split by whether the step that ends the gap was 'inline' // (same warm process as the step before it — pure framework overhead) or // a 'queue-hop' (first step of a fresh process — dispatch + reinit cost). @@ -838,6 +864,10 @@ describe('workflow benchmarks', () => { }, scenarios: SCENARIO_DESCRIPTIONS, metrics: metricRows, + // Only present when the sequential-steps scenario ran; omitted entirely + // (rather than `[]`) so a result file from another scenario selection + // doesn't grow a meaningless empty field. + ...(sequentialRuns.length > 0 ? { sequentialRuns } : {}), }; fs.writeFileSync(outputPath, JSON.stringify(results, null, 2)); console.log(`[bench] Results written to ${outputPath}`); diff --git a/workbench/nextjs-turbopack/app/api/bench/route.ts b/workbench/nextjs-turbopack/app/api/bench/route.ts index 8a6f91e83b..1bca3e6436 100644 --- a/workbench/nextjs-turbopack/app/api/bench/route.ts +++ b/workbench/nextjs-turbopack/app/api/bench/route.ts @@ -1,3 +1,4 @@ +import { trace } from '@opentelemetry/api'; import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; import { start } from 'workflow/api'; @@ -57,7 +58,13 @@ export async function POST(request: NextRequest) { const clientStart = Date.now(); // @ts-expect-error - arbitrary call to a dynamically resolved workflow const run = await start(fn, args); - return NextResponse.json({ runId: run.runId, clientStart }); + // Surface this request's trace id so the runner can link a benchmark run + // back to its Datadog trace from the PR comment, instead of anyone having + // to hunt for it by deployment id / time window. The span is the one + // @vercel/otel opened for this route invocation (see instrumentation.ts), + // and it propagates into the workflow's own spans. + const traceId = trace.getActiveSpan()?.spanContext().traceId; + return NextResponse.json({ runId: run.runId, clientStart, traceId }); } catch (error) { return NextResponse.json( { From 9f917ed6295fec16a83563ab109f34a2d79e8232 Mon Sep 17 00:00:00 2001 From: "vercel[bot]" <35613825+vercel[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:00:39 +0000 Subject: [PATCH 2/4] Log the run/trace links instead of rendering them into the comment The run id and Datadog trace are debugging aids, not part of the benchmark's reported result, so they belong in the job's own output rather than in the PR comment body. Logging them where the runs are produced also makes them available in two cases the comment could never cover: a local `pnpm bench`, and a job that fails before the comment step runs. This drops the comment-rendering side entirely -- `renderSequentialRunLinks`, the `baselineSequentialRuns` baseline plumbing in `annotateWithBaseline`, and the `sequentialRuns` field on the result artifact, which existed only to carry the data to the renderer. The `main`-baseline side of the link goes away with it: which run produced the baseline histogram is only knowable at comment-render time, where the two artifacts are matched. Co-Authored-By: Claude Opus 5 Co-Authored-By: shalabhc --- .changeset/bench-histogram-run-trace-links.md | 2 +- .github/scripts/render-benchmark-comment.mjs | 53 +----------- .../scripts/render-benchmark-comment.test.js | 81 +------------------ packages/core/e2e/benchmark.test.ts | 39 +++++---- .../nextjs-turbopack/app/api/bench/route.ts | 6 +- 5 files changed, 28 insertions(+), 153 deletions(-) diff --git a/.changeset/bench-histogram-run-trace-links.md b/.changeset/bench-histogram-run-trace-links.md index fccdb09937..ef8164eed6 100644 --- a/.changeset/bench-histogram-run-trace-links.md +++ b/.changeset/bench-histogram-run-trace-links.md @@ -1,4 +1,4 @@ --- --- -Record the run id and Datadog trace id of each sequential-steps benchmark iteration and link them under the STSO histograms in the benchmark PR comment, so a suspicious distribution can be opened in APM directly. +Log the run id and Datadog trace id of each sequential-steps benchmark iteration, so a suspicious STSO distribution can be opened in APM directly instead of hunted down by deployment id and time window. diff --git a/.github/scripts/render-benchmark-comment.mjs b/.github/scripts/render-benchmark-comment.mjs index 9d975086ac..a601422ae0 100644 --- a/.github/scripts/render-benchmark-comment.mjs +++ b/.github/scripts/render-benchmark-comment.mjs @@ -217,14 +217,8 @@ export function annotateWithBaseline(results, baseline) { const methodology = (result) => result.methodologyVersion ?? 'legacy'; const keyFor = (result, row) => `${methodology(result)}/${result.backend}/${result.app}/${row.metric}/${row.scenario}`; - // Same key minus the metric row — the sequential-run identities annotated - // below live on the result, not on an individual metric row. - const resultKeyFor = (result) => - `${methodology(result)}/${result.backend}/${result.app}`; const baselineRows = new Map(); - const baselineResults = new Map(); for (const result of baseline) { - baselineResults.set(resultKeyFor(result), result); for (const row of result.metrics ?? []) { baselineRows.set(keyFor(result, row), row); } @@ -243,18 +237,10 @@ export function annotateWithBaseline(results, baseline) { if (Array.isArray(base.raw)) annotated.baselineRaw = base.raw; return annotated; }; - return results.map((result) => { - // The baseline's sequential runs (when it recorded them) let the histogram - // section link both sides of the diff, not just this run's. - const baseResult = baselineResults.get(resultKeyFor(result)); - return { - ...result, - metrics: (result.metrics ?? []).map((row) => annotate(result, row)), - ...(Array.isArray(baseResult?.sequentialRuns) - ? { baselineSequentialRuns: baseResult.sequentialRuns } - : {}), - }; - }); + return results.map((result) => ({ + ...result, + metrics: (result.metrics ?? []).map((row) => annotate(result, row)), + })); } // ============================================================================ @@ -465,36 +451,6 @@ function renderStsoRowDiff(row) { return lines.join('\n'); } -// Datadog APM permalink for a trace id. The benchmark deployment exports its -// OTel spans to Datadog, and `/api/bench` returns the trace id of the request -// that started each run (see packages/core/e2e/benchmark.test.ts). -const DATADOG_TRACE_URL = 'https://app.datadoghq.com/apm/trace/'; - -/** - * Renders the identity of the sequential-steps run(s) behind the histograms - * above — run id plus a Datadog trace link for each, for this run and for the - * `main` run it is diffed against. Both histograms (inline and queue-hop) come - * out of the same iteration, so this is one line for the whole section rather - * than a repeat under each chart. - * - * Everything is optional: a deployment built before `/api/bench` returned a - * trace id yields a bare run id, and a `main` baseline from before this landed - * has no runs to name at all, in which case only this run's side is shown. - */ -function renderSequentialRunLinks(result) { - const current = result.sequentialRuns ?? []; - const baseline = result.baselineSequentialRuns ?? []; - if (current.length === 0 && baseline.length === 0) return []; - const formatRun = (run) => - `\`${run.runId}\`${run.traceId ? ` ([trace](${DATADOG_TRACE_URL}${run.traceId}))` : ''}`; - const side = (label, runs) => - runs.length > 0 ? `${label}: ${runs.map(formatRun).join(', ')}` : undefined; - const sides = [side('this run', current), side('`main`', baseline)].filter( - Boolean - ); - return ['', `Runs behind these histograms — ${sides.join(' · ')}`]; -} - /** * Renders a per-scenario histogram diff (bucketed step counts) and a * cumulative-time diff (sum of all STSO samples) against `main`, for every @@ -519,7 +475,6 @@ function renderStsoDiffSection(result) { `📈 STSO distribution${anyBaseline ? ' vs main' : ''} (inline / queue-hop histograms)`, '', ...rows.map(renderStsoRowDiff), - ...renderSequentialRunLinks(result), '', '', ].join('\n'); diff --git a/.github/scripts/render-benchmark-comment.test.js b/.github/scripts/render-benchmark-comment.test.js index ff763ea1ba..2782be6d42 100644 --- a/.github/scripts/render-benchmark-comment.test.js +++ b/.github/scripts/render-benchmark-comment.test.js @@ -461,7 +461,7 @@ test('CLI fails when completed with no results', async () => { /** A sequential-steps result whose STSO rows carry raw samples, as the * benchmark runner now records them (every gap, not a sampled window). */ -function sequentialResult({ inline, queueHop, sequentialRuns }) { +function sequentialResult({ inline, queueHop }) { const stsoRow = (scenario, raw) => ({ metric: 'stso', scenario, @@ -482,7 +482,6 @@ function sequentialResult({ inline, queueHop, sequentialRuns }) { stsoRow('1020 steps (inline)', inline), stsoRow('1020 steps (queue-hop)', queueHop), ], - ...(sequentialRuns ? { sequentialRuns } : {}), }); } @@ -600,84 +599,6 @@ test('strips raw samples from the embedded history data block', async () => { assert.doesNotMatch(rerendered, /STSO distribution/); }); -test('links the run id and Datadog trace under the histograms', async () => { - const { renderComment } = await loadModule(); - const body = renderComment({ - status: 'completed', - results: [ - sequentialResult({ - inline: [160, 360], - queueHop: [2100], - sequentialRuns: [{ runId: 'run_this', traceId: 'abc123' }], - }), - ], - baseline: [ - sequentialResult({ - inline: [160, 360], - queueHop: [2100], - sequentialRuns: [{ runId: 'run_main', traceId: 'def456' }], - }), - ], - history: [], - commit: 'abcdef1234567890', - }); - - // One line for the whole section — both histograms come from the same run. - assert.match( - body, - /Runs behind these histograms — this run: `run_this` \(\[trace\]\(https:\/\/app\.datadoghq\.com\/apm\/trace\/abc123\)\) · `main`: `run_main` \(\[trace\]\(https:\/\/app\.datadoghq\.com\/apm\/trace\/def456\)\)<\/sub>/ - ); - // ...and it sits inside the collapsed distribution section, below the charts. - const section = body.slice( - body.indexOf('📈 STSO distribution'), - body.indexOf('', body.indexOf('📈 STSO distribution')) - ); - assert.ok( - section.indexOf('Runs behind these histograms') > - section.lastIndexOf('```'), - 'run links should render after the last bar chart' - ); -}); - -test('degrades to bare run ids when no trace or baseline runs exist', async () => { - const { renderComment } = await loadModule(); - const body = renderComment({ - status: 'completed', - results: [ - sequentialResult({ - inline: [160, 360], - queueHop: [2100], - // A deployment built before /api/bench reported a trace id. - sequentialRuns: [{ runId: 'run_this' }], - }), - ], - // A `main` baseline from before the runner recorded run identities. - baseline: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], - history: [], - commit: 'abcdef1234567890', - }); - - assert.match( - body, - /Runs behind these histograms — this run: `run_this`<\/sub>/ - ); - assert.doesNotMatch(body, /datadoghq/); -}); - -test('omits the run links entirely when no run identities were recorded', async () => { - const { renderComment } = await loadModule(); - const body = renderComment({ - status: 'completed', - results: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], - baseline: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], - history: [], - commit: 'abcdef1234567890', - }); - - assert.match(body, /📈 STSO distribution vs main/); - assert.doesNotMatch(body, /Runs behind these histograms/); -}); - test('buckets negative STSO gaps separately from the slow tail', async () => { const { renderComment } = await loadModule(); // Consecutive step timestamps come from different step bodies, so a gap can diff --git a/packages/core/e2e/benchmark.test.ts b/packages/core/e2e/benchmark.test.ts index 3da53354e0..4f522c9a64 100644 --- a/packages/core/e2e/benchmark.test.ts +++ b/packages/core/e2e/benchmark.test.ts @@ -280,8 +280,8 @@ async function triggerBenchRun( return { runId: data.runId, clientStart: data.clientStart, - // Optional: a deployment built before the route reported it simply omits - // the trace link from the rendered comment. + // Optional: a deployment built before the route reported it simply logs + // the run id without a trace link. traceId: typeof data.traceId === 'string' ? data.traceId : undefined, }; } @@ -573,17 +573,6 @@ interface MetricRow extends MetricStats { const metricRows: MetricRow[] = []; -/** Identity of one sequential-steps iteration. */ -interface SequentialRunRef { - runId: string; - traceId?: string; -} - -/** The runs behind the STSO histograms, recorded so the PR comment can link - * straight to each run and its Datadog trace — which is where a - * suspicious-looking distribution has to be investigated. */ -const sequentialRuns: SequentialRunRef[] = []; - function recordMetric( metric: string, scenario: string, @@ -659,6 +648,11 @@ const SCENARIO_DESCRIPTIONS = [ }, ]; +// Datadog APM permalink for a trace id. The benchmark deployment exports its +// OTel spans to Datadog, and `/api/bench` returns the trace id of the request +// that started each run. +const DATADOG_TRACE_URL = 'https://app.datadoghq.com/apm/trace/'; + describe('workflow benchmarks', () => { // Preflight: prove the deployment executes workflows (and the trigger route // works) before any scenario spends its attempt budget. Without this, a @@ -800,9 +794,18 @@ describe('workflow benchmarks', () => { extraAttempts: Math.max(2, Math.ceil(SEQUENTIAL_ITERATIONS * 0.5)), } ); - sequentialRuns.push( - ...results.map((r) => ({ runId: r.runId, traceId: r.traceId })) - ); + // Name the runs behind the STSO histograms in this job's own log, right + // where they were produced. When a bucket looks wrong the investigation + // starts in APM, and this saves the usual hunt by deployment id + time + // window. Logged rather than rendered into the PR comment so it is also + // there for a local `pnpm bench` and for a run whose comment step never + // gets to execute. + for (const { runId, traceId } of results) { + console.log( + `[bench] ${SCENARIO_SEQUENTIAL} run ${runId}` + + (traceId ? ` — trace ${DATADOG_TRACE_URL}${traceId}` : '') + ); + } // Report STSO split by whether the step that ends the gap was 'inline' // (same warm process as the step before it — pure framework overhead) or // a 'queue-hop' (first step of a fresh process — dispatch + reinit cost). @@ -864,10 +867,6 @@ describe('workflow benchmarks', () => { }, scenarios: SCENARIO_DESCRIPTIONS, metrics: metricRows, - // Only present when the sequential-steps scenario ran; omitted entirely - // (rather than `[]`) so a result file from another scenario selection - // doesn't grow a meaningless empty field. - ...(sequentialRuns.length > 0 ? { sequentialRuns } : {}), }; fs.writeFileSync(outputPath, JSON.stringify(results, null, 2)); console.log(`[bench] Results written to ${outputPath}`); diff --git a/workbench/nextjs-turbopack/app/api/bench/route.ts b/workbench/nextjs-turbopack/app/api/bench/route.ts index 1bca3e6436..f1e9b9c874 100644 --- a/workbench/nextjs-turbopack/app/api/bench/route.ts +++ b/workbench/nextjs-turbopack/app/api/bench/route.ts @@ -58,9 +58,9 @@ export async function POST(request: NextRequest) { const clientStart = Date.now(); // @ts-expect-error - arbitrary call to a dynamically resolved workflow const run = await start(fn, args); - // Surface this request's trace id so the runner can link a benchmark run - // back to its Datadog trace from the PR comment, instead of anyone having - // to hunt for it by deployment id / time window. The span is the one + // Surface this request's trace id so the runner can log a benchmark run's + // Datadog trace next to its run id, instead of anyone having to hunt for it + // by deployment id / time window. The span is the one // @vercel/otel opened for this route invocation (see instrumentation.ts), // and it propagates into the workflow's own spans. const traceId = trace.getActiveSpan()?.spanContext().traceId; From 3686956cfe4a5688d900f04755db2c5ca4695fa0 Mon Sep 17 00:00:00 2001 From: "vercel[bot]" <35613825+vercel[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:00:40 +0000 Subject: [PATCH 3/4] Say what the trigger trace actually contains under linked mode The route comment claimed the trigger request's span "propagates into the workflow's own spans". That only holds under WORKFLOW_TRACE_MODE=continuous. Nothing in the workbench or benchmarks.yml sets the mode, so the benchmark deployment runs the default `linked` (packages/core/src/telemetry.ts), where each workflow/step invocation is its own trace root and the trigger's trace carries `workflow.start` plus span links out to those roots. The logged link is still the right entry point -- one hop through the links, which Datadog renders -- but the comment should describe that, so nobody opening a trigger-only trace while debugging a histogram concludes the run produced no spans. Raised by @TooTallNate in review of #3248. Co-Authored-By: Claude Opus 5 Co-Authored-By: Shalabh Chaturvedi <7066873+shalabhc@users.noreply.github.com> Co-Authored-By: shalabhc --- workbench/nextjs-turbopack/app/api/bench/route.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/workbench/nextjs-turbopack/app/api/bench/route.ts b/workbench/nextjs-turbopack/app/api/bench/route.ts index f1e9b9c874..f6e5771c91 100644 --- a/workbench/nextjs-turbopack/app/api/bench/route.ts +++ b/workbench/nextjs-turbopack/app/api/bench/route.ts @@ -60,9 +60,15 @@ export async function POST(request: NextRequest) { const run = await start(fn, args); // Surface this request's trace id so the runner can log a benchmark run's // Datadog trace next to its run id, instead of anyone having to hunt for it - // by deployment id / time window. The span is the one - // @vercel/otel opened for this route invocation (see instrumentation.ts), - // and it propagates into the workflow's own spans. + // by deployment id / time window. The span is the one @vercel/otel opened + // for this route invocation (see instrumentation.ts). + // + // Under the default WORKFLOW_TRACE_MODE=linked -- which is what this + // deployment runs, since nothing sets the mode -- that trace is NOT the + // whole run: each workflow/step invocation is its own trace root, and this + // one holds the `workflow.start` span plus span links out to those roots. + // So it is an entry point to the run (one hop through the links, which + // Datadog renders), not a single trace containing every step's spans. const traceId = trace.getActiveSpan()?.spanContext().traceId; return NextResponse.json({ runId: run.runId, clientStart, traceId }); } catch (error) { From 6896fb3b3bdeb739152ad07231a09d93a5e06a13 Mon Sep 17 00:00:00 2001 From: "vercel[bot]" <35613825+vercel[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:00:40 +0000 Subject: [PATCH 4/4] Log a Datadog span search alongside the trigger trace link Under the default linked trace mode the trigger's trace holds only `workflow.start` plus span links, so opening it lands one hop away from the spans an STSO investigation needs. Log an APM search on `@workflow.run.id:` next to it, which goes straight to the run's execution spans. Both links are logged rather than one replacing the other: the search depends on `workflow.run.id` being an indexed span tag in the org, and the permalink works regardless. Suggested by @TooTallNate in review of #3248. Co-Authored-By: Claude Opus 5 Co-Authored-By: Shalabh Chaturvedi <7066873+shalabhc@users.noreply.github.com> Co-Authored-By: shalabhc --- .changeset/bench-histogram-run-trace-links.md | 2 +- packages/core/e2e/benchmark.test.ts | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.changeset/bench-histogram-run-trace-links.md b/.changeset/bench-histogram-run-trace-links.md index ef8164eed6..94b370657f 100644 --- a/.changeset/bench-histogram-run-trace-links.md +++ b/.changeset/bench-histogram-run-trace-links.md @@ -1,4 +1,4 @@ --- --- -Log the run id and Datadog trace id of each sequential-steps benchmark iteration, so a suspicious STSO distribution can be opened in APM directly instead of hunted down by deployment id and time window. +Log the run id of each sequential-steps benchmark iteration alongside two Datadog APM links — the trigger request's trace and a span search for the run — so a suspicious STSO distribution can be opened in APM directly instead of hunted down by deployment id and time window. diff --git a/packages/core/e2e/benchmark.test.ts b/packages/core/e2e/benchmark.test.ts index 4f522c9a64..8baf436bb7 100644 --- a/packages/core/e2e/benchmark.test.ts +++ b/packages/core/e2e/benchmark.test.ts @@ -653,6 +653,24 @@ const SCENARIO_DESCRIPTIONS = [ // that started each run. const DATADOG_TRACE_URL = 'https://app.datadoghq.com/apm/trace/'; +/** + * Datadog APM search for the spans tagged with a given `workflow.run.id`. + * + * The permalink above opens the *trigger's* trace, which under the default + * `WORKFLOW_TRACE_MODE=linked` holds only `workflow.start` plus span links out + * to the per-invocation trace roots — an entry point to the run rather than the + * run itself. This search lands straight on the execution spans, which is where + * an STSO investigation actually goes, so both are logged. + * + * Depends on `workflow.run.id` being an indexed span tag in the Datadog org. If + * it isn't, this returns an empty search and the trace permalink stays the way + * in; neither link is load-bearing for the benchmark itself. + */ +function datadogRunSearchUrl(runId: string): string { + const query = encodeURIComponent(`@workflow.run.id:${runId}`); + return `https://app.datadoghq.com/apm/traces?query=${query}`; +} + describe('workflow benchmarks', () => { // Preflight: prove the deployment executes workflows (and the trigger route // works) before any scenario spends its attempt budget. Without this, a @@ -803,7 +821,8 @@ describe('workflow benchmarks', () => { for (const { runId, traceId } of results) { console.log( `[bench] ${SCENARIO_SEQUENTIAL} run ${runId}` + - (traceId ? ` — trace ${DATADOG_TRACE_URL}${traceId}` : '') + (traceId ? ` — trace ${DATADOG_TRACE_URL}${traceId}` : '') + + ` — spans ${datadogRunSearchUrl(runId)}` ); } // Report STSO split by whether the step that ends the gap was 'inline'