From 7a7ecc8b1ce697db4f481f3bc3f278e0b548b9c0 Mon Sep 17 00:00:00 2001 From: jshah Date: Wed, 30 Sep 2026 15:46:21 -0500 Subject: [PATCH 1/3] feat: capture and persist per-sample cache_read tokens Map provider cached_tokens into usage, accumulate across turns/solvers, and write run- and sample-level cache_read columns in parquet so per-task cost can be computed from harness artifacts. --- src/benchmarks/agent-cli/harness.ts | 1 + src/benchmarks/deep-swe/solver.ts | 1 + src/benchmarks/draco/benchmark.test.ts | 1 + src/benchmarks/harbor/agent-loop.ts | 4 +++ src/benchmarks/mmlu-pro.test.ts | 2 ++ .../search/browsecomp/benchmark.test.ts | 1 + src/benchmarks/search/core/usage.test.ts | 14 +++++++++ src/benchmarks/search/core/usage.ts | 2 ++ src/benchmarks/search/dsqa/benchmark.test.ts | 2 ++ src/benchmarks/search/hle/benchmark.test.ts | 1 + .../search/widesearch/benchmark.test.ts | 3 ++ src/benchmarks/swe-atlas/solver.ts | 1 + src/benchmarks/swe-bench/solver.ts | 1 + src/benchmarks/tau-bench-airline/solver.ts | 3 ++ src/benchmarks/tau3-bench-banking/solver.ts | 3 ++ .../terminal-bench/ori-parquet.test.ts | 1 + .../terminal-bench/ori-solver.test.ts | 4 +++ src/benchmarks/terminal-bench/ori-solver.ts | 1 + src/benchmarks/vgi-bench/benchmark.test.ts | 1 + src/benchmarks/wandr/scorer.test.ts | 1 + src/harness/core.ts | 2 ++ src/harness/metric.ts | 10 +++++- src/harness/progress.ts | 1 + src/harness/run.ts | 23 +++++++++----- src/providers/responses-client.test.ts | 31 +++++++++++++++++++ src/providers/responses-client.ts | 19 ++++++++++++ src/results/parquet-schema.ts | 6 ++++ src/results/parquet.test.ts | 2 ++ src/results/parquet.ts | 25 +++++++++++++++ src/runtime/generation-resolver.test.ts | 3 ++ src/runtime/generation-resolver.ts | 5 +++ src/server/dashboard.test.ts | 1 + src/server/run-summary.ts | 2 ++ 33 files changed, 170 insertions(+), 8 deletions(-) diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index c25003ae..53dc2504 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -704,6 +704,7 @@ function parseJsonAgentStream(stdout: string): OriAgentRun { inputTokens: inputTokens + cacheRead + cacheWrite, outputTokens, totalTokens, + cacheReadTokens: cacheRead, reasoningTokens, totalCost, } diff --git a/src/benchmarks/deep-swe/solver.ts b/src/benchmarks/deep-swe/solver.ts index e7e29d34..2dada069 100644 --- a/src/benchmarks/deep-swe/solver.ts +++ b/src/benchmarks/deep-swe/solver.ts @@ -472,6 +472,7 @@ const ZERO_AGENT_USAGE: ModelUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }; diff --git a/src/benchmarks/draco/benchmark.test.ts b/src/benchmarks/draco/benchmark.test.ts index 2efd31ea..a5ae33e3 100644 --- a/src/benchmarks/draco/benchmark.test.ts +++ b/src/benchmarks/draco/benchmark.test.ts @@ -35,6 +35,7 @@ function runResultWith( inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/harbor/agent-loop.ts b/src/benchmarks/harbor/agent-loop.ts index a76654b6..0435657b 100644 --- a/src/benchmarks/harbor/agent-loop.ts +++ b/src/benchmarks/harbor/agent-loop.ts @@ -438,6 +438,7 @@ export interface UsageAccumulator { inputTokens: number; outputTokens: number; totalTokens: number; + cacheReadTokens: number; reasoningTokens: number; totalCost: number; webSearchRequests: number; @@ -451,6 +452,7 @@ function newUsageAccumulator(): UsageAccumulator { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, webSearchRequests: 0, @@ -467,6 +469,7 @@ function addUsage(acc: UsageAccumulator, usage: ModelUsage | undefined): void { acc.inputTokens += usage.inputTokens ?? 0; acc.outputTokens += usage.outputTokens ?? 0; acc.totalTokens += usage.totalTokens ?? 0; + acc.cacheReadTokens += usage.cacheReadTokens ?? 0; acc.reasoningTokens += usage.reasoningTokens ?? 0; acc.totalCost += usage.totalCost ?? 0; const serverToolUse = usage.serverToolUse; @@ -492,6 +495,7 @@ function toModelUsage(acc: UsageAccumulator): ModelUsage { inputTokens: acc.inputTokens, outputTokens: acc.outputTokens, totalTokens: acc.totalTokens, + cacheReadTokens: acc.cacheReadTokens, reasoningTokens: acc.reasoningTokens, totalCost: acc.totalCost, serverToolUse: acc.seenServerToolUse diff --git a/src/benchmarks/mmlu-pro.test.ts b/src/benchmarks/mmlu-pro.test.ts index 2f474337..94788f16 100644 --- a/src/benchmarks/mmlu-pro.test.ts +++ b/src/benchmarks/mmlu-pro.test.ts @@ -47,6 +47,7 @@ function runResultWith( inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, @@ -144,6 +145,7 @@ describe("MMLU-Pro runLevelScores", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/search/browsecomp/benchmark.test.ts b/src/benchmarks/search/browsecomp/benchmark.test.ts index 1a52be50..4891f426 100644 --- a/src/benchmarks/search/browsecomp/benchmark.test.ts +++ b/src/benchmarks/search/browsecomp/benchmark.test.ts @@ -308,6 +308,7 @@ describe("browseCompRunLevelScores", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/search/core/usage.test.ts b/src/benchmarks/search/core/usage.test.ts index ab837005..69c90ec7 100644 --- a/src/benchmarks/search/core/usage.test.ts +++ b/src/benchmarks/search/core/usage.test.ts @@ -9,6 +9,7 @@ describe("mergeModelUsages", () => { inputTokens: 10, outputTokens: 5, totalTokens: 15, + cacheReadTokens: 0, reasoningTokens: 2, totalCost: 0.02, serverToolUse: { @@ -22,6 +23,7 @@ describe("mergeModelUsages", () => { inputTokens: 4, outputTokens: 2, totalTokens: 6, + cacheReadTokens: 0, reasoningTokens: 1, totalCost: 0.001, serverToolUse: { @@ -36,6 +38,7 @@ describe("mergeModelUsages", () => { inputTokens: 14, outputTokens: 7, totalTokens: 24, + cacheReadTokens: 0, reasoningTokens: 3, totalCost: 0.021, serverToolUse: { @@ -45,6 +48,17 @@ describe("mergeModelUsages", () => { }, }); }); + it("sums cacheReadTokens when present", () => { + expect( + mergeModelUsages([ + { inputTokens: 10, cacheReadTokens: 4 }, + { inputTokens: 5, cacheReadTokens: 2 }, + ]) + ).toEqual({ + inputTokens: 15, + cacheReadTokens: 6, + }); + }); it("returns undefined when no call reports usage", () => { expect(mergeModelUsages([undefined, undefined])).toBeUndefined(); }); diff --git a/src/benchmarks/search/core/usage.ts b/src/benchmarks/search/core/usage.ts index f54312c2..15aa0bfa 100644 --- a/src/benchmarks/search/core/usage.ts +++ b/src/benchmarks/search/core/usage.ts @@ -43,6 +43,7 @@ export function mergeModelUsages( const inputTokens = sum(defined.map((usage) => usage.inputTokens)); const outputTokens = sum(defined.map((usage) => usage.outputTokens)); const totalTokens = sum(defined.map((usage) => usage.totalTokens)); + const cacheReadTokens = sum(defined.map((usage) => usage.cacheReadTokens)); const reasoningTokens = sum(defined.map((usage) => usage.reasoningTokens)); const totalCost = sum(defined.map((usage) => usage.totalCost)); const serverToolUse = mergeServerToolUse( @@ -54,6 +55,7 @@ export function mergeModelUsages( inputTokens, outputTokens, totalTokens, + cacheReadTokens, reasoningTokens, totalCost, serverToolUse, diff --git a/src/benchmarks/search/dsqa/benchmark.test.ts b/src/benchmarks/search/dsqa/benchmark.test.ts index 0996051c..3b63cf60 100644 --- a/src/benchmarks/search/dsqa/benchmark.test.ts +++ b/src/benchmarks/search/dsqa/benchmark.test.ts @@ -232,6 +232,7 @@ describe("DSQA benchmark", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, @@ -334,6 +335,7 @@ describe("DSQA metrics", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/search/hle/benchmark.test.ts b/src/benchmarks/search/hle/benchmark.test.ts index 7ddae287..7bde5b8d 100644 --- a/src/benchmarks/search/hle/benchmark.test.ts +++ b/src/benchmarks/search/hle/benchmark.test.ts @@ -293,6 +293,7 @@ function runResult(sampleScores: SampleScore[]): RunResult { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/search/widesearch/benchmark.test.ts b/src/benchmarks/search/widesearch/benchmark.test.ts index 2d87fa95..a9622935 100644 --- a/src/benchmarks/search/widesearch/benchmark.test.ts +++ b/src/benchmarks/search/widesearch/benchmark.test.ts @@ -242,6 +242,7 @@ describe("wideSearchScorer and run-level scores", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, @@ -268,6 +269,7 @@ describe("wideSearchScorer and run-level scores", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, @@ -338,6 +340,7 @@ describe("wideSearchScorer and run-level scores", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/swe-atlas/solver.ts b/src/benchmarks/swe-atlas/solver.ts index bd3571d8..7adda32f 100644 --- a/src/benchmarks/swe-atlas/solver.ts +++ b/src/benchmarks/swe-atlas/solver.ts @@ -54,6 +54,7 @@ const ZERO_AGENT_USAGE: ModelUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }; diff --git a/src/benchmarks/swe-bench/solver.ts b/src/benchmarks/swe-bench/solver.ts index 7c2e066a..e9c13026 100644 --- a/src/benchmarks/swe-bench/solver.ts +++ b/src/benchmarks/swe-bench/solver.ts @@ -400,6 +400,7 @@ const ZERO_AGENT_USAGE: ModelUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }; diff --git a/src/benchmarks/tau-bench-airline/solver.ts b/src/benchmarks/tau-bench-airline/solver.ts index c2adf9f1..b89e906d 100644 --- a/src/benchmarks/tau-bench-airline/solver.ts +++ b/src/benchmarks/tau-bench-airline/solver.ts @@ -106,6 +106,7 @@ export function airlineSolver({ inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }; @@ -157,6 +158,7 @@ export function airlineSolver({ accUsage.inputTokens += output.usage.inputTokens ?? 0; accUsage.outputTokens += output.usage.outputTokens ?? 0; accUsage.totalTokens += output.usage.totalTokens ?? 0; + accUsage.cacheReadTokens += output.usage.cacheReadTokens ?? 0; accUsage.reasoningTokens += output.usage.reasoningTokens ?? 0; accUsage.totalCost += output.usage.totalCost ?? 0; } @@ -205,6 +207,7 @@ export function airlineSolver({ inputTokens: accUsage.inputTokens, outputTokens: accUsage.outputTokens, totalTokens: accUsage.totalTokens, + cacheReadTokens: accUsage.cacheReadTokens, reasoningTokens: accUsage.reasoningTokens, totalCost: accUsage.totalCost, }; diff --git a/src/benchmarks/tau3-bench-banking/solver.ts b/src/benchmarks/tau3-bench-banking/solver.ts index 0cbd7aba..a67bdd6f 100644 --- a/src/benchmarks/tau3-bench-banking/solver.ts +++ b/src/benchmarks/tau3-bench-banking/solver.ts @@ -226,6 +226,7 @@ export function bankingSolver({ inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }; @@ -299,6 +300,7 @@ export function bankingSolver({ accUsage.inputTokens += output.usage.inputTokens ?? 0; accUsage.outputTokens += output.usage.outputTokens ?? 0; accUsage.totalTokens += output.usage.totalTokens ?? 0; + accUsage.cacheReadTokens += output.usage.cacheReadTokens ?? 0; accUsage.reasoningTokens += output.usage.reasoningTokens ?? 0; accUsage.totalCost += output.usage.totalCost ?? 0; } @@ -341,6 +343,7 @@ export function bankingSolver({ inputTokens: accUsage.inputTokens, outputTokens: accUsage.outputTokens, totalTokens: accUsage.totalTokens, + cacheReadTokens: accUsage.cacheReadTokens, reasoningTokens: accUsage.reasoningTokens, totalCost: accUsage.totalCost, }; diff --git a/src/benchmarks/terminal-bench/ori-parquet.test.ts b/src/benchmarks/terminal-bench/ori-parquet.test.ts index d769a17b..f9343b69 100644 --- a/src/benchmarks/terminal-bench/ori-parquet.test.ts +++ b/src/benchmarks/terminal-bench/ori-parquet.test.ts @@ -134,6 +134,7 @@ async function runOriSampleToParquetRow() { inputTokens: usage?.inputTokens ?? 0, outputTokens: usage?.outputTokens ?? 0, totalTokens: usage?.totalTokens ?? 0, + cacheReadTokens: 0, reasoningTokens: usage?.reasoningTokens ?? 0, totalCost: usage?.totalCost ?? 0, generationTimeMs: state.output?.generationTimeMs ?? 0, diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index cb466033..17caaa29 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -1274,6 +1274,7 @@ describe("terminal-bench Prime Agent via ori", () => { inputTokens: 4494, outputTokens: 3, totalTokens: 4497, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0.0018023999999999998, }); @@ -1540,6 +1541,7 @@ describe("terminal-bench Prime Agent via ori", () => { inputTokens: 37, outputTokens: 8, totalTokens: 126, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }); @@ -1619,6 +1621,7 @@ describe("terminal-bench omp via ori", () => { cacheRead: 0, cacheWrite: 0, totalTokens: 127, + cacheReadTokens: 0, reasoningTokens: 4, cost: { input: 0.000048, @@ -1696,6 +1699,7 @@ describe("terminal-bench omp via ori", () => { inputTokens: 120, outputTokens: 7, totalTokens: 127, + cacheReadTokens: 0, reasoningTokens: 4, totalCost: 0.0000536, }); diff --git a/src/benchmarks/terminal-bench/ori-solver.ts b/src/benchmarks/terminal-bench/ori-solver.ts index a6fcd525..783a7b3d 100644 --- a/src/benchmarks/terminal-bench/ori-solver.ts +++ b/src/benchmarks/terminal-bench/ori-solver.ts @@ -26,6 +26,7 @@ const ZERO_USAGE: ModelUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, }; diff --git a/src/benchmarks/vgi-bench/benchmark.test.ts b/src/benchmarks/vgi-bench/benchmark.test.ts index a300b66d..17b608fc 100644 --- a/src/benchmarks/vgi-bench/benchmark.test.ts +++ b/src/benchmarks/vgi-bench/benchmark.test.ts @@ -375,6 +375,7 @@ describe("VGI-Bench registry", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/benchmarks/wandr/scorer.test.ts b/src/benchmarks/wandr/scorer.test.ts index d03cd3f8..3503f617 100644 --- a/src/benchmarks/wandr/scorer.test.ts +++ b/src/benchmarks/wandr/scorer.test.ts @@ -29,6 +29,7 @@ function resultWith(values: readonly number[]): RunResult { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/harness/core.ts b/src/harness/core.ts index 6dbb16fd..927e000e 100644 --- a/src/harness/core.ts +++ b/src/harness/core.ts @@ -136,6 +136,8 @@ export interface UsageTotals { readonly inputTokens: number; readonly outputTokens: number; readonly totalTokens: number; + /** Cache-read tokens; subset of inputTokens when the provider reports it. */ + readonly cacheReadTokens: number; readonly reasoningTokens: number; readonly totalCost: number; readonly generationTimeMs: number; diff --git a/src/harness/metric.ts b/src/harness/metric.ts index bec4e063..f41d074b 100644 --- a/src/harness/metric.ts +++ b/src/harness/metric.ts @@ -1,4 +1,10 @@ -import type { ModelMessage, ResponseItem, Score, ScoreValue } from "./core"; +import type { + ModelMessage, + ModelUsage, + ResponseItem, + Score, + ScoreValue, +} from "./core"; import { ScoreValue as SV, scoreToNumber } from "./core"; export interface SampleScore { @@ -10,6 +16,8 @@ export interface SampleScore { readonly requestBody?: Readonly>; readonly generationIds?: readonly string[]; readonly generationTimeMs?: number; + /** Per-sample (task×epoch) model usage, including cache-read tokens when present. */ + readonly usage?: ModelUsage; readonly metadata?: Readonly>; readonly input?: string; readonly target?: string; diff --git a/src/harness/progress.ts b/src/harness/progress.ts index 9adfae5e..b6a95ffd 100644 --- a/src/harness/progress.ts +++ b/src/harness/progress.ts @@ -129,6 +129,7 @@ export const CheckpointDataSchema = z.object({ inputTokens: z.number(), outputTokens: z.number(), totalTokens: z.number(), + cacheReadTokens: z.number().optional(), reasoningTokens: z.number(), totalCost: z.number(), webSearchRequests: z.number().optional(), diff --git a/src/harness/run.ts b/src/harness/run.ts index fcda382a..07708b32 100644 --- a/src/harness/run.ts +++ b/src/harness/run.ts @@ -162,6 +162,7 @@ function accumulateOutcome( inputTokens: acc.usage.inputTokens + (u?.inputTokens ?? 0), outputTokens: acc.usage.outputTokens + (u?.outputTokens ?? 0), totalTokens: acc.usage.totalTokens + (u?.totalTokens ?? 0), + cacheReadTokens: acc.usage.cacheReadTokens + (u?.cacheReadTokens ?? 0), reasoningTokens: acc.usage.reasoningTokens + (u?.reasoningTokens ?? 0), totalCost: acc.usage.totalCost + (u?.totalCost ?? 0), generationTimeMs: acc.usage.generationTimeMs + (item.generationTimeMs ?? 0), @@ -185,15 +186,21 @@ function applyReplayedUsage( return outcome; } const u = outcome.usage; + const usage = { + ...u, + inputTokens: (u?.inputTokens ?? 0) + replayed.inputTokens, + outputTokens: (u?.outputTokens ?? 0) + replayed.outputTokens, + totalTokens: (u?.totalTokens ?? 0) + replayed.totalTokens, + cacheReadTokens: (u?.cacheReadTokens ?? 0) + replayed.cacheReadTokens, + reasoningTokens: (u?.reasoningTokens ?? 0) + replayed.reasoningTokens, + totalCost: (u?.totalCost ?? 0) + replayed.totalCost, + }; return { ...outcome, - usage: { - ...u, - inputTokens: (u?.inputTokens ?? 0) + replayed.inputTokens, - outputTokens: (u?.outputTokens ?? 0) + replayed.outputTokens, - totalTokens: (u?.totalTokens ?? 0) + replayed.totalTokens, - reasoningTokens: (u?.reasoningTokens ?? 0) + replayed.reasoningTokens, - totalCost: (u?.totalCost ?? 0) + replayed.totalCost, + usage, + sampleScore: { + ...outcome.sampleScore, + usage, }, generationTimeMs: (outcome.generationTimeMs ?? 0) + replayed.generationTimeMs, @@ -228,6 +235,7 @@ function evaluateOne( responseItems: state.responseItems, requestBody: state.requestBody, generationTimeMs: state.output?.generationTimeMs, + usage: state.output?.usage, metadata: state.sample.metadata, input: sample.input, target: sample.target.text, @@ -334,6 +342,7 @@ const ZERO_USAGE: UsageTotals = { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/providers/responses-client.test.ts b/src/providers/responses-client.test.ts index 2f5f3ac2..dafc313b 100644 --- a/src/providers/responses-client.test.ts +++ b/src/providers/responses-client.test.ts @@ -405,6 +405,36 @@ describe("usageFromResponses", () => { reasoningTokens: 2, }); }); + it("maps input_tokens_details.cached_tokens as cacheReadTokens", () => { + expect( + usageFromResponses({ + input_tokens: 100, + output_tokens: 20, + total_tokens: 120, + input_tokens_details: { cached_tokens: 40 }, + }) + ).toEqual({ + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + cacheReadTokens: 40, + }); + }); + it("maps camelCase inputTokensDetails.cachedTokens", () => { + expect( + usageFromResponses({ + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + inputTokensDetails: { cachedTokens: 25 }, + }) + ).toEqual({ + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + cacheReadTokens: 25, + }); + }); it("returns undefined for null usage", () => { expect(usageFromResponses(null)).toBeUndefined(); }); @@ -446,6 +476,7 @@ describe("usageFromResponses", () => { inputTokens: 770, outputTokens: 186, totalTokens: 956, + cacheReadTokens: 0, reasoningTokens: 64, totalCost: 0.0046999, serverToolUse: { toolCallsRequested: 1, toolCallsExecuted: 1 }, diff --git a/src/providers/responses-client.ts b/src/providers/responses-client.ts index 6678991d..b56fd263 100644 --- a/src/providers/responses-client.ts +++ b/src/providers/responses-client.ts @@ -545,6 +545,12 @@ export function usageFromResponses( const detailsRaw = usage["outputTokensDetails"] ?? usage["output_tokens_details"]; const details = isRecord(detailsRaw) ? detailsRaw : undefined; + const inputDetailsRaw = + usage["inputTokensDetails"] ?? + usage["input_tokens_details"] ?? + usage["promptTokensDetails"] ?? + usage["prompt_tokens_details"]; + const inputDetails = isRecord(inputDetailsRaw) ? inputDetailsRaw : undefined; const inputTokens = numField(usage, "inputTokens", "input_tokens"); const outputTokens = numField(usage, "outputTokens", "output_tokens"); const totalTokens = numField(usage, "totalTokens", "total_tokens"); @@ -552,6 +558,18 @@ export function usageFromResponses( details !== undefined ? numField(details, "reasoningTokens", "reasoning_tokens") : undefined; + const cacheReadTokens = + inputDetails !== undefined + ? numField(inputDetails, "cachedTokens", "cached_tokens") + : undefined; + const cacheReadFromTopLevel = numField( + usage, + "cacheReadTokens", + "cache_read_tokens", + "cache_read_input_tokens", + "cachedTokens", + "cached_tokens" + ); const totalCost = numField(usage, "cost"); const serverToolUseRaw = usage["serverToolUseDetails"] ?? usage["server_tool_use_details"]; @@ -578,6 +596,7 @@ export function usageFromResponses( inputTokens, outputTokens, totalTokens, + cacheReadTokens: cacheReadTokens ?? cacheReadFromTopLevel, reasoningTokens, totalCost, serverToolUse: hasServerToolUse diff --git a/src/results/parquet-schema.ts b/src/results/parquet-schema.ts index 38c30219..44c1d11f 100644 --- a/src/results/parquet-schema.ts +++ b/src/results/parquet-schema.ts @@ -23,6 +23,7 @@ export const BenchmarkResultRowSchema = z.object({ input_tokens: z.number(), output_tokens: z.number(), total_tokens: z.number(), + cache_read_tokens: z.number().default(0), reasoning_tokens: z.number(), total_cost: z.number(), generation_time_ms: z.number(), @@ -33,6 +34,11 @@ export const BenchmarkResultRowSchema = z.object({ sample_id: z.string(), epoch: z.number(), sample_generation_time_ms: z.number().nullish(), + sample_input_tokens: z.number().nullish(), + sample_output_tokens: z.number().nullish(), + sample_total_tokens: z.number().nullish(), + sample_cache_read_tokens: z.number().nullish(), + sample_reasoning_tokens: z.number().nullish(), input: z.string().nullable(), target: z.string().nullable(), score_value: z.string(), diff --git a/src/results/parquet.test.ts b/src/results/parquet.test.ts index a819802d..734fa916 100644 --- a/src/results/parquet.test.ts +++ b/src/results/parquet.test.ts @@ -28,6 +28,7 @@ const USAGE = { inputTokens: 100, outputTokens: 50, totalTokens: 150, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0.01, generationTimeMs: 1000, @@ -139,6 +140,7 @@ describe("runResultToParquet", () => { expect(row.input_tokens).toBe(100); expect(row.output_tokens).toBe(50); expect(row.total_tokens).toBe(150); + expect(row.cache_read_tokens).toBe(0); expect(row.reasoning_tokens).toBe(0); expect(row.total_cost).toBe(0.01); expect(row.generation_time_ms).toBe(1000); diff --git a/src/results/parquet.ts b/src/results/parquet.ts index 51a0422e..4f9594cb 100644 --- a/src/results/parquet.ts +++ b/src/results/parquet.ts @@ -73,6 +73,7 @@ const COLUMN_SPECS = [ { name: "input_tokens", type: "INT32", nullable: false }, { name: "output_tokens", type: "INT32", nullable: false }, { name: "total_tokens", type: "INT32", nullable: false }, + { name: "cache_read_tokens", type: "INT32", nullable: false }, { name: "reasoning_tokens", type: "INT32", nullable: false }, { name: "total_cost", type: "DOUBLE", nullable: false }, { name: "generation_time_ms", type: "INT32", nullable: false }, @@ -83,6 +84,11 @@ const COLUMN_SPECS = [ { name: "sample_id", type: "STRING", nullable: false }, { name: "epoch", type: "INT32", nullable: false }, { name: "sample_generation_time_ms", type: "INT32", nullable: true }, + { name: "sample_input_tokens", type: "INT32", nullable: true }, + { name: "sample_output_tokens", type: "INT32", nullable: true }, + { name: "sample_total_tokens", type: "INT32", nullable: true }, + { name: "sample_cache_read_tokens", type: "INT32", nullable: true }, + { name: "sample_reasoning_tokens", type: "INT32", nullable: true }, { name: "input", type: "STRING", nullable: true }, { name: "target", type: "STRING", nullable: true }, { name: "score_value", type: "STRING", nullable: false }, @@ -188,6 +194,9 @@ function cellValue(name: ColumnName, ctx: RowContext, s: SampleScore): unknown { case "total_tokens": { return ctx.usage.totalTokens; } + case "cache_read_tokens": { + return ctx.usage.cacheReadTokens; + } case "reasoning_tokens": { return ctx.usage.reasoningTokens; } @@ -218,6 +227,21 @@ function cellValue(name: ColumnName, ctx: RowContext, s: SampleScore): unknown { case "sample_generation_time_ms": { return s.generationTimeMs ?? null; } + case "sample_input_tokens": { + return s.usage?.inputTokens ?? null; + } + case "sample_output_tokens": { + return s.usage?.outputTokens ?? null; + } + case "sample_total_tokens": { + return s.usage?.totalTokens ?? null; + } + case "sample_cache_read_tokens": { + return s.usage?.cacheReadTokens ?? null; + } + case "sample_reasoning_tokens": { + return s.usage?.reasoningTokens ?? null; + } case "input": { return s.input ?? null; } @@ -453,6 +477,7 @@ export function summarizeChunkRows( inputTokens: first.input_tokens, outputTokens: first.output_tokens, totalTokens: first.total_tokens, + cacheReadTokens: first.cache_read_tokens, reasoningTokens: first.reasoning_tokens, totalCost: first.total_cost, generationTimeMs: first.generation_time_ms, diff --git a/src/runtime/generation-resolver.test.ts b/src/runtime/generation-resolver.test.ts index d6064e56..0d2def4f 100644 --- a/src/runtime/generation-resolver.test.ts +++ b/src/runtime/generation-resolver.test.ts @@ -46,6 +46,7 @@ const SOURCE_USAGE: ReplayedUsage = { inputTokens: 10, outputTokens: 25, totalTokens: 35, + cacheReadTokens: 0, reasoningTokens: 5, totalCost: 0.0015, generationTimeMs: 1200, @@ -93,6 +94,7 @@ describe("resolveCollectedGenerations", () => { inputTokens: 20, outputTokens: 50, totalTokens: 70, + cacheReadTokens: 0, reasoningTokens: 10, totalCost: 0.003, generationTimeMs: 2400, @@ -356,6 +358,7 @@ describe("makeOpenRouterGenerationResolver", () => { inputTokens: 0, outputTokens: 0, totalTokens: 0, + cacheReadTokens: 0, reasoningTokens: 0, totalCost: 0, generationTimeMs: 0, diff --git a/src/runtime/generation-resolver.ts b/src/runtime/generation-resolver.ts index 06a8906e..3ffb3af1 100644 --- a/src/runtime/generation-resolver.ts +++ b/src/runtime/generation-resolver.ts @@ -30,6 +30,7 @@ export interface ReplayedUsage { readonly inputTokens: number; readonly outputTokens: number; readonly totalTokens: number; + readonly cacheReadTokens: number; readonly reasoningTokens: number; readonly totalCost: number; readonly generationTimeMs: number; @@ -57,6 +58,8 @@ const GenerationLookupSchema = z.object({ native_tokens_prompt: z.number().nullish(), native_tokens_completion: z.number().nullish(), native_tokens_reasoning: z.number().nullish(), + native_tokens_cached: z.number().nullish(), + native_tokens_cache: z.number().nullish(), total_cost: z.number().nullish(), generation_time: z.number().nullish(), }), @@ -109,6 +112,7 @@ function usageFromLookup( inputTokens, outputTokens, totalTokens: inputTokens + outputTokens, + cacheReadTokens: data.native_tokens_cached ?? data.native_tokens_cache ?? 0, reasoningTokens: data.native_tokens_reasoning ?? 0, totalCost: data.total_cost ?? 0, generationTimeMs: data.generation_time ?? 0, @@ -286,6 +290,7 @@ function sumReplayedUsage( inputTokens: acc.inputTokens + usage.inputTokens, outputTokens: acc.outputTokens + usage.outputTokens, totalTokens: acc.totalTokens + usage.totalTokens, + cacheReadTokens: acc.cacheReadTokens + usage.cacheReadTokens, reasoningTokens: acc.reasoningTokens + usage.reasoningTokens, totalCost: acc.totalCost + usage.totalCost, generationTimeMs: acc.generationTimeMs + usage.generationTimeMs, diff --git a/src/server/dashboard.test.ts b/src/server/dashboard.test.ts index 7d2ad06d..2cd5a3ca 100644 --- a/src/server/dashboard.test.ts +++ b/src/server/dashboard.test.ts @@ -1118,6 +1118,7 @@ describe("benchmark runs dashboard", () => { inputTokens: 10, outputTokens: 5, totalTokens: 15, + cacheReadTokens: 0, reasoningTokens: 2, totalCost: 0.01, generationTimeMs: 100, diff --git a/src/server/run-summary.ts b/src/server/run-summary.ts index 2506c491..7566efe2 100644 --- a/src/server/run-summary.ts +++ b/src/server/run-summary.ts @@ -31,6 +31,7 @@ export interface DetailedRunSummary { readonly inputTokens: number; readonly outputTokens: number; readonly totalTokens: number; + readonly cacheReadTokens: number; readonly reasoningTokens: number; readonly totalCost: number; readonly generationTimeMs: number; @@ -144,6 +145,7 @@ export function summarizeRunRows( inputTokens: summary.inputTokens, outputTokens: summary.outputTokens, totalTokens: summary.totalTokens, + cacheReadTokens: summary.cacheReadTokens, reasoningTokens: summary.reasoningTokens, totalCost: summary.totalCost, generationTimeMs: summary.generationTimeMs, From 48bce49aee5d5f76a7ff6288d484a6ef780c13ae Mon Sep 17 00:00:00 2001 From: jshah Date: Wed, 30 Sep 2026 16:18:10 -0500 Subject: [PATCH 2/3] fix: drop JSDoc comments that fail oxlint --- src/harness/core.ts | 1 - src/harness/metric.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/harness/core.ts b/src/harness/core.ts index 927e000e..6f11653c 100644 --- a/src/harness/core.ts +++ b/src/harness/core.ts @@ -136,7 +136,6 @@ export interface UsageTotals { readonly inputTokens: number; readonly outputTokens: number; readonly totalTokens: number; - /** Cache-read tokens; subset of inputTokens when the provider reports it. */ readonly cacheReadTokens: number; readonly reasoningTokens: number; readonly totalCost: number; diff --git a/src/harness/metric.ts b/src/harness/metric.ts index f41d074b..f1501765 100644 --- a/src/harness/metric.ts +++ b/src/harness/metric.ts @@ -16,7 +16,6 @@ export interface SampleScore { readonly requestBody?: Readonly>; readonly generationIds?: readonly string[]; readonly generationTimeMs?: number; - /** Per-sample (task×epoch) model usage, including cache-read tokens when present. */ readonly usage?: ModelUsage; readonly metadata?: Readonly>; readonly input?: string; From 5b71dcc3861c480eac03aa8144d24c6506eab0cd Mon Sep 17 00:00:00 2001 From: jshah Date: Wed, 30 Sep 2026 16:32:29 -0500 Subject: [PATCH 3/3] fix: wire cache_read through ori usage and parquet tests --- src/benchmarks/agent-cli/harness.ts | 4 +++- src/benchmarks/terminal-bench/ori-parquet.test.ts | 3 ++- src/benchmarks/terminal-bench/ori-solver.test.ts | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 53dc2504..84109240 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -227,10 +227,11 @@ function reasoningFromContent(content: unknown): string | undefined { function usageFromResult(result: Record): ModelUsage { const { usage } = result; + const cacheReadTokens = numberField(usage, "cache_read_input_tokens"); const inputTokens = numberField(usage, "input_tokens") + numberField(usage, "cache_creation_input_tokens") + - numberField(usage, "cache_read_input_tokens"); + cacheReadTokens; const outputTokens = numberField(usage, "output_tokens"); const details = isRecord(usage) ? usage["output_tokens_details"] : undefined; const reasoningTokens = numberField(details, "thinking_tokens"); @@ -243,6 +244,7 @@ function usageFromResult(result: Record): ModelUsage { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens, + cacheReadTokens, reasoningTokens, totalCost: numberField(result, "total_cost_usd"), ...definedValues({ diff --git a/src/benchmarks/terminal-bench/ori-parquet.test.ts b/src/benchmarks/terminal-bench/ori-parquet.test.ts index f9343b69..a2502f4e 100644 --- a/src/benchmarks/terminal-bench/ori-parquet.test.ts +++ b/src/benchmarks/terminal-bench/ori-parquet.test.ts @@ -134,7 +134,7 @@ async function runOriSampleToParquetRow() { inputTokens: usage?.inputTokens ?? 0, outputTokens: usage?.outputTokens ?? 0, totalTokens: usage?.totalTokens ?? 0, - cacheReadTokens: 0, + cacheReadTokens: usage?.cacheReadTokens ?? 0, reasoningTokens: usage?.reasoningTokens ?? 0, totalCost: usage?.totalCost ?? 0, generationTimeMs: state.output?.generationTimeMs ?? 0, @@ -145,6 +145,7 @@ async function runOriSampleToParquetRow() { epoch: 0, score, messages: state.messages, + ...(usage !== undefined && { usage }), ...(state.responseItems !== undefined && { responseItems: state.responseItems, }), diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index 17caaa29..7c41eb93 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -1541,7 +1541,7 @@ describe("terminal-bench Prime Agent via ori", () => { inputTokens: 37, outputTokens: 8, totalTokens: 126, - cacheReadTokens: 0, + cacheReadTokens: 10, reasoningTokens: 0, totalCost: 0, });