diff --git a/.changeset/services-series.md b/.changeset/services-series.md new file mode 100644 index 00000000..26faac4b --- /dev/null +++ b/.changeset/services-series.md @@ -0,0 +1,11 @@ +--- +"@bounded-systems/prx": minor +--- + +Add `prx services series` — effort/token series showing completion rate by +model × spend tier. + +Reveals the non-monotonic relationship between spend and outcome (higher spend +does not monotonically improve completion rate past a model-specific peak). +Refactors the shared work-unit loading into a private `buildWuidOutcomes` +helper used by both the diamond and series projectors. diff --git a/packages/prx/src/pr-state/cli.ts b/packages/prx/src/pr-state/cli.ts index 619644ef..163c2354 100644 --- a/packages/prx/src/pr-state/cli.ts +++ b/packages/prx/src/pr-state/cli.ts @@ -337,7 +337,7 @@ import { runDedupeBd as doctorRunDedupeBd } from "../doctor/dedupe-bd.ts"; // GH-1823: audit actor — read-only adherence metrics over the artifact graph. import { runAuditIngest, runAuditUow, runAuditSystem } from "../audit/cli.ts"; // GH-1407: services actor — Anthropic prompt-cache hit-rate projector. -import { runServicesDiamond, runServicesStatus } from "../services/cli.ts"; +import { runServicesDiamond, runServicesSeries, runServicesStatus } from "../services/cli.ts"; import { taskRoleMachine, taskRoles, type TaskRole } from "../machine/machines/task.ts"; import { workflowMachine } from "../machine/machines/workflow.ts"; import { resolveWorktreePath } from "../tools/worktree_path.ts"; @@ -1286,6 +1286,12 @@ type ParsedCommand = window?: string | undefined; format: "plain" | "json"; } + | { + // prx-b2n — `prx services series` shows completion rate by model × spend tier. + command: "services-series"; + window?: string | undefined; + format: "plain" | "json"; + } | { // GH-1194: per-actor dispatch envelope. The leading source flag is // injected by normalizeNamespaceArgv (or by intake/implement passthrough); @@ -5790,7 +5796,7 @@ function parseTestGateCommand(rest: string[]): ParsedCommand { } // GH-1407 — `prx services ` read-only external-plane status verb. -const SERVICES_VERBS = ["status", "diamond"] as const; +const SERVICES_VERBS = ["status", "diamond", "series"] as const; function printServicesHelp(): string { return [ @@ -5803,11 +5809,16 @@ function printServicesHelp(): string { " diamond [--window=Nd] [--format=plain|json]", " Cost-vs-outcome diamond: avg spend and completion rate per model.", "", + " series [--window=Nd] [--format=plain|json]", + " Effort/token series: completion rate by model × spend tier.", + " Reveals the non-monotonic relationship between spend and outcome.", + "", "Examples:", " prx services status --anthropic --by=model", " prx services status --anthropic --window=7d --format=json", " prx services diamond --window=30d", - " prx services diamond --format=json", + " prx services series --window=30d", + " prx services series --format=json", ].join("\n"); } @@ -5831,7 +5842,7 @@ function parseServicesCommand(rest: string[]): ParsedCommand { printServicesHelpAndExit(); } - if (verbArg === "diamond") { + if (verbArg === "diamond" || verbArg === "series") { const { values, positionals } = parseArgs({ args: subRest, options: { @@ -5842,13 +5853,12 @@ function parseServicesCommand(rest: string[]): ParsedCommand { allowPositionals: true, }); if (positionals.length > 0) { - throw new CliError("prx services diamond takes no positional arguments"); + throw new CliError(`prx services ${verbArg} takes no positional arguments`); } - return { - command: "services-diamond", - ...(values.window !== undefined ? { window: values.window } : {}), - format: ensureChoice(values.format, ["plain", "json"], "--format"), - }; + const fmt = ensureChoice(values.format, ["plain", "json"], "--format"); + const win = values.window !== undefined ? { window: values.window } : {}; + if (verbArg === "series") return { command: "services-series", ...win, format: fmt }; + return { command: "services-diamond", ...win, format: fmt }; } // verb === "status" @@ -18786,6 +18796,17 @@ export function runCli( ); } + // prx-b2n: effort/token series. + if (parsed.command === "services-series") { + return runServicesSeries( + { + ...(parsed.window !== undefined ? { window: parsed.window } : {}), + format: parsed.format, + }, + output, + ); + } + if (parsed.command === "intake-mirror") { const handler = deps.runIntakeMirror ?? runIntakeMirror; const validated: IntakeMirrorOptions = intakeMirrorOptionsSchema.parse({ diff --git a/packages/prx/src/services/anthropic.ts b/packages/prx/src/services/anthropic.ts index 7126f599..ab38cb94 100644 --- a/packages/prx/src/services/anthropic.ts +++ b/packages/prx/src/services/anthropic.ts @@ -92,12 +92,23 @@ const CLOSED_STATES = new Set(["closed"]); type TransitionRow = { issue: string; state_to: string; ts: string }; -export function projectAnthropicDiamond( - db: Database, - opts: AnthropicProjectorOptions = {}, -): DiamondPoint[] { - const since = opts.since; +type WuidOutcome = { + dominantModel: string; + totalCost: number; + totalCacheRead: number; + totalInput: number; + outcome: "completed" | "closed" | "in_progress"; +}; +/** + * Shared loader used by both the diamond and series projectors. + * + * For each work unit that has at least one attributed usage event, resolves + * the dominant model (highest cost share) and the outcome from the latest + * transition. Work units with no `workUnitId` are skipped — there is no + * outcome to measure for unattached sessions. + */ +function buildWuidOutcomes(db: Database, since?: string): WuidOutcome[] { const rows = since ? db .query( @@ -108,9 +119,6 @@ export function projectAnthropicDiamond( .query(`SELECT raw_json FROM events WHERE action = 'non-interactive-agent'`) .all(); - // Accumulate cost per (workUnitId, model). Each work unit is later assigned - // to its dominant model (highest cost share) so it contributes exactly one - // point to the diamond — no double-counting across models. type ModelAccum = { cost: number; cache_read: number; input: number }; const byWuid = new Map>(); @@ -123,7 +131,7 @@ export function projectAnthropicDiamond( } if (payload.subkind !== "usage") continue; const wuid = payload.workUnitId; - if (!wuid) continue; // unattached — no outcome to measure + if (!wuid) continue; const model = payload.model ?? "(unknown model)"; const inner = byWuid.get(wuid) ?? new Map(); const entry = inner.get(model) ?? { cost: 0, cache_read: 0, input: 0 }; @@ -136,8 +144,6 @@ export function projectAnthropicDiamond( if (byWuid.size === 0) return []; - // Get latest state_to per issue from the transitions log (ascending order so - // each later row overwrites the previous — last write wins = most recent). const allTransitions = db .query( `SELECT issue, state_to, ts FROM transitions ORDER BY ts ASC`, @@ -148,19 +154,8 @@ export function projectAnthropicDiamond( if (t.issue) latestState.set(t.issue, t.state_to); } - type ModelAgg = { - work_units: number; - total_cost: number; - completed: number; - closed: number; - in_progress: number; - total_cache_read: number; - total_input: number; - }; - const modelAgg = new Map(); - + const outcomes: WuidOutcome[] = []; for (const [wuid, modelMap] of byWuid) { - // Dominant model = highest cost share for this work unit. let dominantModel = "(unknown model)"; let maxCost = -1; let totalCost = 0; @@ -175,7 +170,6 @@ export function projectAnthropicDiamond( dominantModel = model; } } - const state = latestState.get(wuid); const outcome = state ? COMPLETED_STATES.has(state) @@ -184,7 +178,30 @@ export function projectAnthropicDiamond( ? "closed" : "in_progress" : "in_progress"; + outcomes.push({ dominantModel, totalCost, totalCacheRead, totalInput, outcome }); + } + return outcomes; +} + +export function projectAnthropicDiamond( + db: Database, + opts: AnthropicProjectorOptions = {}, +): DiamondPoint[] { + const outcomes = buildWuidOutcomes(db, opts.since); + if (outcomes.length === 0) return []; + + type ModelAgg = { + work_units: number; + total_cost: number; + completed: number; + closed: number; + in_progress: number; + total_cache_read: number; + total_input: number; + }; + const modelAgg = new Map(); + for (const { dominantModel, totalCost, totalCacheRead, totalInput, outcome } of outcomes) { const agg = modelAgg.get(dominantModel) ?? { work_units: 0, total_cost: 0, @@ -222,6 +239,100 @@ export function projectAnthropicDiamond( .sort((a, b) => b.completion_rate - a.completion_rate || b.total_cost_usd - a.total_cost_usd); } +/** + * One point on the effort/token series — a (model, cost_tier) pair showing + * completion rate within a specific spend band. + * + * The series exposes the non-monotonic relationship between spend and outcome: + * more tokens ≠ better results past a model-specific peak. Use it to pick the + * cost-efficient tier for each task rather than defaulting to max effort. + */ +export type SeriesPoint = { + model: string; + tier: string; + tier_min: number; + tier_max: number; + work_units: number; + avg_cost_usd: number; + completion_rate: number; + hit_rate: number; +}; + +// Cost bands chosen to capture the ~$8 non-monotonic peak observed in +// Opus 4.8 (peaks around $8, dips past $10.5 — higher spend ≠ better outcome). +const COST_TIERS: ReadonlyArray<{ label: string; min: number; max: number }> = [ + { label: "<$2", min: 0, max: 2 }, + { label: "$2–$5", min: 2, max: 5 }, + { label: "$5–$10", min: 5, max: 10 }, + { label: "$10–$20", min: 10, max: 20 }, + { label: "$20–$40", min: 20, max: 40 }, + { label: "$40+", min: 40, max: Infinity }, +]; + +function costTier(cost: number): (typeof COST_TIERS)[number] { + for (const tier of COST_TIERS) { + if (cost < tier.max) return tier; + } + return COST_TIERS[COST_TIERS.length - 1]!; +} + +export function projectAnthropicSeries( + db: Database, + opts: AnthropicProjectorOptions = {}, +): SeriesPoint[] { + const outcomes = buildWuidOutcomes(db, opts.since); + if (outcomes.length === 0) return []; + + type TierAgg = { + work_units: number; + total_cost: number; + completed: number; + total_cache_read: number; + total_input: number; + }; + // key = `${model}\x00${tierLabel}` + const tierAgg = new Map(); + + for (const { dominantModel, totalCost, totalCacheRead, totalInput, outcome } of outcomes) { + const tier = costTier(totalCost); + const key = `${dominantModel}\x00${tier.label}`; + const agg = tierAgg.get(key) ?? { + work_units: 0, + total_cost: 0, + completed: 0, + total_cache_read: 0, + total_input: 0, + tier, + }; + agg.work_units += 1; + agg.total_cost += totalCost; + agg.total_cache_read += totalCacheRead; + agg.total_input += totalInput; + if (outcome === "completed") agg.completed += 1; + tierAgg.set(key, agg); + } + + return [...tierAgg.entries()] + .map(([key, agg]) => { + const model = key.split("\x00")[0]!; + const denom = agg.total_input + agg.total_cache_read; + return { + model, + tier: agg.tier.label, + tier_min: agg.tier.min, + tier_max: agg.tier.max === Infinity ? -1 : agg.tier.max, + work_units: agg.work_units, + avg_cost_usd: agg.work_units > 0 ? agg.total_cost / agg.work_units : 0, + completion_rate: agg.work_units > 0 ? agg.completed / agg.work_units : 0, + hit_rate: denom === 0 ? 0 : agg.total_cache_read / denom, + }; + }) + .sort((a, b) => { + const modelCmp = a.model.localeCompare(b.model); + return modelCmp !== 0 ? modelCmp : a.tier_min - b.tier_min; + }); +} + export function projectAnthropicUsage( db: Database, opts: AnthropicProjectorOptions = {}, diff --git a/packages/prx/src/services/cli.ts b/packages/prx/src/services/cli.ts index 95567113..bcdaece9 100644 --- a/packages/prx/src/services/cli.ts +++ b/packages/prx/src/services/cli.ts @@ -11,11 +11,13 @@ import { openAuditDb } from "../audit/store/db.ts"; import { ingestAuditSources } from "../audit/store/ingest.ts"; import { projectAnthropicDiamond, + projectAnthropicSeries, projectAnthropicUsage, resolveWindowFloor, type AnthropicProjectorBy, type AnthropicUsageBucket, type DiamondPoint, + type SeriesPoint, } from "./anthropic.ts"; export type ServicesOutput = { @@ -162,6 +164,76 @@ function renderDiamond( } } +export type RunServicesSeriesOptions = { + window?: string; + format?: "plain" | "json"; +}; + +export function runServicesSeries( + opts: RunServicesSeriesOptions, + output: ServicesOutput, + deps: ServicesCliDeps = {}, +): number { + const db = + deps.db ?? + (deps.openDb ?? openAuditDb)({ + stateDirOverride: deps.stateDirOverride, + }); + + if (deps.auditDir || deps.transitionDir) { + ingestAuditSources(db, { + ...(deps.auditDir ? { auditDir: deps.auditDir } : {}), + ...(deps.transitionDir ? { transitionDir: deps.transitionDir } : {}), + }); + } + + const since = resolveWindowFloor(opts.window, deps.now?.() ?? new Date()); + const points = projectAnthropicSeries(db, { ...(since ? { since } : {}) }); + + if (opts.format === "json") { + output.log( + JSON.stringify({ + plane: "anthropic", + window: opts.window ?? null, + since: since ?? null, + points, + }), + ); + } else { + renderSeries(points, opts, output); + } + + if (!deps.db) db.close(); + return 0; +} + +function renderSeries( + points: SeriesPoint[], + opts: RunServicesSeriesOptions, + output: ServicesOutput, +): void { + const windowLabel = opts.window ? ` (window=${opts.window})` : ""; + output.log(`anthropic services series — completion rate by model × spend tier${windowLabel}`); + if (points.length === 0) { + output.log( + " (no work-unit cost data found — try `prx audit ingest` with --audit-dir and --transition-dir)", + ); + return; + } + output.log( + ` ${"model".padEnd(28)} ${"tier".padStart(10)} ${"wus".padStart(5)} ${"avg_cost".padStart(10)} ${"completion".padStart(11)}`, + ); + let lastModel = ""; + for (const p of points) { + if (p.model !== lastModel && lastModel !== "") output.log(""); + lastModel = p.model; + const completionPct = (p.completion_rate * 100).toFixed(1) + "%"; + output.log( + ` ${p.model.padEnd(28)} ${p.tier.padStart(10)} ${String(p.work_units).padStart(5)} ${("$" + p.avg_cost_usd.toFixed(2)).padStart(10)} ${completionPct.padStart(11)}`, + ); + } +} + function renderText( buckets: AnthropicUsageBucket[], opts: RunServicesStatusOptions, diff --git a/packages/prx/test/services/anthropic.test.ts b/packages/prx/test/services/anthropic.test.ts index aaeeb8b7..6e2251a1 100644 --- a/packages/prx/test/services/anthropic.test.ts +++ b/packages/prx/test/services/anthropic.test.ts @@ -5,6 +5,7 @@ import { describe, expect, test } from "bun:test"; import { projectAnthropicDiamond, + projectAnthropicSeries, projectAnthropicUsage, resolveWindowFloor, } from "../../src/services/anthropic.ts"; @@ -102,6 +103,73 @@ function seedNonUsage(db: Database, ts: string): void { ); } +describe("projectAnthropicSeries", () => { + function makeDbWithTransitions(): Database { + const db = makeDb(); + db.exec(` + CREATE TABLE transitions ( + id TEXT PRIMARY KEY, issue TEXT, state_from TEXT NOT NULL, + state_to TEXT NOT NULL, actor TEXT NOT NULL, artifact TEXT, + ts TEXT NOT NULL, proof_commit TEXT, proof_checks_json TEXT + ); + `); + return db; + } + + function seedTransition(db: Database, issue: string, state_to: string, ts: string): void { + db.run( + `INSERT INTO transitions (id, issue, state_from, state_to, actor, ts) VALUES (?, ?, ?, ?, ?, ?)`, + [`tr::${issue}::${ts}`, issue, "open", state_to, "test", ts], + ); + } + + test("buckets work units by cost tier within each model", () => { + const db = makeDbWithTransitions(); + // GH-1: $1.50 → <$2 tier, merged + seedUsage(db, { ts: "2026-05-15T00:00:00Z", workUnitId: "GH-1", model: "claude-opus-4-8", total_cost_usd: 1.5, input_tokens: 10, cache_read_input_tokens: 90 }); + seedTransition(db, "GH-1", "merged", "2026-05-15T01:00:00Z"); + // GH-2: $7.00 → $5–$10 tier, merged + seedUsage(db, { ts: "2026-05-15T00:01:00Z", workUnitId: "GH-2", model: "claude-opus-4-8", total_cost_usd: 7.0, input_tokens: 50, cache_read_input_tokens: 400 }); + seedTransition(db, "GH-2", "merged", "2026-05-15T02:00:00Z"); + // GH-3: $12.00 → $10–$20 tier, closed (no completion) + seedUsage(db, { ts: "2026-05-15T00:02:00Z", workUnitId: "GH-3", model: "claude-opus-4-8", total_cost_usd: 12.0, input_tokens: 80, cache_read_input_tokens: 600 }); + seedTransition(db, "GH-3", "closed", "2026-05-15T03:00:00Z"); + + const points = projectAnthropicSeries(db); + expect(points.every((p) => p.model === "claude-opus-4-8")).toBe(true); + + const cheap = points.find((p) => p.tier === "<$2")!; + expect(cheap.work_units).toBe(1); + expect(cheap.completion_rate).toBe(1); + + const mid = points.find((p) => p.tier === "$5–$10")!; + expect(mid.work_units).toBe(1); + expect(mid.completion_rate).toBe(1); + + const expensive = points.find((p) => p.tier === "$10–$20")!; + expect(expensive.work_units).toBe(1); + expect(expensive.completion_rate).toBe(0); + }); + + test("sorts by model then tier_min ascending", () => { + const db = makeDbWithTransitions(); + seedUsage(db, { ts: "2026-05-15T00:00:00Z", workUnitId: "GH-A", model: "claude-opus-4-8", total_cost_usd: 25.0 }); + seedUsage(db, { ts: "2026-05-15T00:01:00Z", workUnitId: "GH-B", model: "claude-opus-4-8", total_cost_usd: 1.0 }); + seedUsage(db, { ts: "2026-05-15T00:02:00Z", workUnitId: "GH-C", model: "claude-haiku-4-5", total_cost_usd: 3.0 }); + + const points = projectAnthropicSeries(db); + const opusPoints = points.filter((p) => p.model === "claude-opus-4-8"); + // cheap tier (<$2) should come before expensive tier ($20–$40) + expect(opusPoints[0]!.tier_min).toBeLessThan(opusPoints[1]!.tier_min); + }); + + test("returns empty when no work units have a workUnitId", () => { + const db = makeDbWithTransitions(); + seedUsage(db, { ts: "2026-05-15T00:00:00Z", input_tokens: 10 }); + expect(projectAnthropicSeries(db)).toEqual([]); + }); +}); + describe("projectAnthropicUsage", () => { test("groups by profile and computes hit_rate against (input + cache_read)", () => { const db = makeDb(); diff --git a/packages/prx/test/services/cli.test.ts b/packages/prx/test/services/cli.test.ts index b885567b..111d4482 100644 --- a/packages/prx/test/services/cli.test.ts +++ b/packages/prx/test/services/cli.test.ts @@ -6,7 +6,7 @@ import { Database } from "bun:sqlite"; import { describe, expect, test } from "bun:test"; -import { runServicesDiamond, runServicesStatus } from "../../src/services/cli.ts"; +import { runServicesDiamond, runServicesSeries, runServicesStatus } from "../../src/services/cli.ts"; function seedDb(): Database { const db = new Database(":memory:"); @@ -160,6 +160,40 @@ describe("runServicesDiamond", () => { }); }); +describe("runServicesSeries", () => { + test("--format=json emits structured points with tier and completion_rate", () => { + const db = seedDbWithTransitions(); + const logs: string[] = []; + const code = runServicesSeries( + { format: "json" }, + { log: (l) => logs.push(l), error: () => {} }, + { db }, + ); + expect(code).toBe(0); + const parsed = JSON.parse(logs[0]!) as { + plane: string; + points: Array<{ model: string; tier: string; completion_rate: number }>; + }; + expect(parsed.plane).toBe("anthropic"); + expect(parsed.points.length).toBeGreaterThan(0); + expect(parsed.points.every((p) => typeof p.tier === "string")).toBe(true); + expect(parsed.points.every((p) => typeof p.completion_rate === "number")).toBe(true); + }); + + test("plain format prints a header and model rows grouped by tier", () => { + const db = seedDbWithTransitions(); + const logs: string[] = []; + const code = runServicesSeries( + { format: "plain" }, + { log: (l) => logs.push(l), error: () => {} }, + { db }, + ); + expect(code).toBe(0); + expect(logs.some((l) => l.includes("series"))).toBe(true); + expect(logs.some((l) => l.includes("claude-opus-4-8") || l.includes("claude-haiku-4-5"))).toBe(true); + }); +}); + describe("runServicesStatus", () => { test("--anthropic --format=json emits a structured envelope per bucket", () => { const db = seedDb();