Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,9 @@ API routes (`packages/app/src/app/api/v1/`):
- `evaluations` — raw `EvalRow[]`
- `server-log` — retrieve benchmark runtime logs
- `invalidate` — invalidate API cache (admin)
- `tco-feed?model=dsv4&workloads=1024x1024,8192x1024&tiers=30,50,75,100&format=csv` — per-hardware Pareto-frontier output-throughput reads at fixed interactivity tiers, for external spreadsheet TCO models (Excel Power Query)
- `tco-feed?model=dsv4&workloads=1024x1024,8192x1024&tiers=30,50,75,100&format=csv` — per-hardware Pareto-frontier output-throughput reads at fixed interactivity tiers, for external spreadsheet TCO models (Excel Power Query); `view=scores` (optional `weights`, `workload_weights`, `alpha`) folds them into one tier-weighted, workload-blended, output-equivalent score per hardware

**API routes return raw DB data** — no presentation logic. Frontend handles all transformations. Sole exception: `tco-feed`, which runs the calculator's frontier interpolation server-side because its consumers (spreadsheets) cannot execute the TS transforms; it serves reads only — weights/assumptions stay with the consumer.
**API routes return raw DB data** — no presentation logic. Frontend handles all transformations. Sole exception: `tco-feed`, which runs the calculator's frontier interpolation server-side because its consumers (spreadsheets) cannot execute the TS transforms; its assumptions (tier weights, workload mix, α) enter only as explicit query params with documented defaults, so a published sheet's URL fully records its methodology.

Static content routes (no DB):

Expand Down
92 changes: 89 additions & 3 deletions packages/app/src/app/api/v1/tco-feed/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,14 @@ import { loadFixture } from '@/lib/test-fixtures';

import {
computeTcoFeed,
computeTcoScores,
parseAlpha,
parseTiers,
parseTierWeights,
parseWorkloads,
parseWorkloadWeights,
tcoFeedToCsv,
tcoScoresToCsv,
type TcoFeedRow,
type TcoFeedSourceRow,
type TcoFeedWorkload,
Expand All @@ -31,9 +36,11 @@ export const dynamic = 'force-dynamic';
* data" convention: it applies the dashboard's frontier interpolation
* (calculator/interpolation.ts) server-side so external consumers get
* numbers identical to what the chart renders, without reimplementing —
* and inevitably drifting from — the spline. Keep all *assumptions*
* (tier weights, workload mix, token-value ratios) out of this route;
* it serves reads, consumers apply weights.
* and inevitably drifting from — the spline. Assumptions (tier weights,
* workload mix, the α token-value ratio) enter ONLY as query params on the
* `scores` view, never as hidden constants beyond the documented defaults —
* a published sheet's URL fully records its methodology, and `view=scores`
* is exactly SUMPRODUCT over the `points` view, reproducible by hand.
*
* Query params (all optional):
* - model — DB key (`dsv4`) or display name (`DeepSeek-V4-Pro`).
Expand All @@ -45,6 +52,21 @@ export const dynamic = 'force-dynamic';
* - date — `YYYY-MM-DD`; reads use data as of this date
* (reproducibility of published sheets). Default: latest.
* - format — `json` (default) or `csv` (one-line Power Query import).
* - view — `points` (default): one row per hardware × workload × tier.
* `scores`: one row per hardware — tier-weighted, workload-
* blended, output-equivalent tok/s/GPU (the single number a
* TCO sheet consumes per chip).
*
* `scores`-only params (400 with `view=points`):
* - weights — per-tier traffic-mix weights, comma-separated,
* normalized to sum 1. Default `0.35,0.4,0.2,0.05`
* for the default tiers, equal weights otherwise.
* - workload_weights — per-workload blend weights, normalized to sum 1.
* Default: equal split. Renormalized over covered
* workloads when a chip lacks data for one.
* - alpha — input-token value ratio in the output-equivalent
* conversion `out × (1 + α × ISL/OSL)`. Default 0.25;
* `alpha=0` scores plain output throughput.
*/

const getCachedTcoFeed = cachedQuery(
Expand Down Expand Up @@ -104,11 +126,75 @@ export async function GET(request: NextRequest) {
return NextResponse.json({ error: 'Invalid format — expected json or csv' }, { status: 400 });
}

const view = params.get('view') ?? 'points';
if (view !== 'points' && view !== 'scores') {
return NextResponse.json(
{ error: 'Invalid view — expected points or scores' },
{ status: 400 },
);
}

const rawWeights = params.get('weights');
const rawWorkloadWeights = params.get('workload_weights');
const rawAlpha = params.get('alpha');
if (view === 'points' && (rawWeights ?? rawWorkloadWeights ?? rawAlpha) !== null) {
// Reject rather than ignore — a consumer passing weights to the points
// view would silently get unweighted numbers.
return NextResponse.json(
{ error: 'weights, workload_weights, and alpha require view=scores' },
{ status: 400 },
);
}

const tierWeights = parseTierWeights(rawWeights, tiers);
if (!tierWeights) {
return NextResponse.json(
{ error: 'Invalid weights — expected one non-negative number per tier, sum > 0' },
{ status: 400 },
);
}

const workloadWeights = parseWorkloadWeights(rawWorkloadWeights, workloads);
if (!workloadWeights) {
return NextResponse.json(
{
error: 'Invalid workload_weights — expected one non-negative number per workload, sum > 0',
},
{ status: 400 },
);
}

const alpha = parseAlpha(rawAlpha);
if (alpha === null) {
return NextResponse.json(
{ error: 'Invalid alpha — expected a number in [0, 10]' },
{ status: 400 },
);
}

try {
const rows = FIXTURES_MODE
? computeTcoFeed(loadFixture<TcoFeedSourceRow[]>('benchmarks'), workloads, tiers)
: await getCachedTcoFeed(dbModelKeys, date, workloads, tiers);

if (view === 'scores') {
const scores = computeTcoScores(rows, workloads, tiers, tierWeights, workloadWeights, alpha);
if (format === 'csv') {
return cachedText(tcoScoresToCsv(scores, workloads), 'text/csv; charset=utf-8');
}
return cachedJson({
model,
db_model_keys: dbModelKeys,
date: date ?? null,
workloads: workloads.map((w) => `${w.isl}x${w.osl}`),
tiers,
weights: tierWeights,
workload_weights: workloadWeights,
alpha,
rows: scores,
});
}

if (format === 'csv') {
return cachedText(tcoFeedToCsv(rows), 'text/csv; charset=utf-8');
}
Expand Down
185 changes: 185 additions & 0 deletions packages/app/src/lib/tco-feed.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,18 @@ import { describe, expect, it } from 'vitest';

import {
computeTcoFeed,
computeTcoScores,
DEFAULT_ALPHA,
DEFAULT_TIER_WEIGHTS,
DEFAULT_TIERS,
DEFAULT_WORKLOADS,
parseAlpha,
parseTiers,
parseTierWeights,
parseWorkloads,
parseWorkloadWeights,
tcoFeedToCsv,
tcoScoresToCsv,
type TcoFeedRow,
type TcoFeedSourceRow,
} from './tco-feed';
Expand Down Expand Up @@ -219,6 +226,11 @@ describe('parseTiers', () => {
expect(parseTiers('10001')).toBeNull();
expect(parseTiers(Array.from({ length: 21 }, (_, i) => String(i + 1)).join(','))).toBeNull();
});

it('rejects duplicate tiers (would double-count in the scores view)', () => {
expect(parseTiers('50,50')).toBeNull();
expect(parseTiers('50,50.0')).toBeNull(); // same numeric value
});
});

describe('parseWorkloads', () => {
Expand All @@ -242,6 +254,68 @@ describe('parseWorkloads', () => {
expect(parseWorkloads('8192x')).toBeNull();
expect(parseWorkloads(Array.from({ length: 9 }, () => '1024x1024').join(','))).toBeNull();
});

it('rejects duplicate workloads (would double-count in the scores view)', () => {
expect(parseWorkloads('8192x1024,8192x1024')).toBeNull();
});
});

describe('parseTierWeights', () => {
it('defaults to the traffic-mix weights for the default tiers', () => {
expect(parseTierWeights(null, DEFAULT_TIERS)).toEqual([...DEFAULT_TIER_WEIGHTS]);
expect(parseTierWeights(' ', [30, 50, 75, 100])).toEqual([...DEFAULT_TIER_WEIGHTS]);
});

it('defaults to equal weights for custom tiers', () => {
expect(parseTierWeights(null, [20, 60])).toEqual([0.5, 0.5]);
});

it('normalizes provided weights to sum 1', () => {
expect(parseTierWeights('2,2,4,2', [30, 50, 75, 100])).toEqual([0.2, 0.2, 0.4, 0.2]);
});

it('rejects count mismatch, negatives, zero sums, and non-numbers', () => {
expect(parseTierWeights('0.5,0.5', [30, 50, 75])).toBeNull();
expect(parseTierWeights('-1,2', [30, 50])).toBeNull();
expect(parseTierWeights('0,0', [30, 50])).toBeNull();
expect(parseTierWeights('a,b', [30, 50])).toBeNull();
expect(parseTierWeights('0.5,,0.5', [30, 50, 75])).toBeNull();
});
});

describe('parseWorkloadWeights', () => {
it('defaults to an equal split', () => {
expect(parseWorkloadWeights(null, [...DEFAULT_WORKLOADS])).toEqual([0.5, 0.5]);
});

it('normalizes provided weights to sum 1', () => {
expect(parseWorkloadWeights('3,1', [...DEFAULT_WORKLOADS])).toEqual([0.75, 0.25]);
});

it('rejects count mismatch, negatives, and zero sums', () => {
expect(parseWorkloadWeights('1', [...DEFAULT_WORKLOADS])).toBeNull();
expect(parseWorkloadWeights('1,-1', [...DEFAULT_WORKLOADS])).toBeNull();
expect(parseWorkloadWeights('0,0', [...DEFAULT_WORKLOADS])).toBeNull();
});
});

describe('parseAlpha', () => {
it('defaults when absent or blank', () => {
expect(parseAlpha(null)).toBe(DEFAULT_ALPHA);
expect(parseAlpha(' ')).toBe(DEFAULT_ALPHA);
});

it('parses valid values including 0 (plain output throughput)', () => {
expect(parseAlpha('0')).toBe(0);
expect(parseAlpha('0.5')).toBe(0.5);
expect(parseAlpha('10')).toBe(10);
});

it('rejects negatives, out-of-range, and non-numbers', () => {
expect(parseAlpha('-0.1')).toBeNull();
expect(parseAlpha('10.1')).toBeNull();
expect(parseAlpha('abc')).toBeNull();
});
});

describe('tcoFeedToCsv', () => {
Expand All @@ -259,3 +333,114 @@ describe('tcoFeedToCsv', () => {
expect(csv.endsWith('\n')).toBe(true);
});
});

const BOTH_WORKLOADS = [
{ isl: 1024, osl: 1024 },
{ isl: 8192, osl: 1024 },
];

/**
* b200 covers both workloads (single-knot frontiers read exactly at tier
* 50); gb300 covers only 8k1k.
*/
function twoWorkloadRows(): TcoFeedSourceRow[] {
return [
makeRow({
hardware: 'b200',
isl: 1024,
osl: 1024,
itl: 1 / 50,
otput: 1000,
date: '2026-05-01',
}),
makeRow({ hardware: 'b200', itl: 1 / 50, otput: 400, date: '2026-07-01' }),
makeRow({ hardware: 'gb300', itl: 1 / 50, otput: 5000 }),
];
}

describe('computeTcoScores', () => {
it('is exactly the weighted sum over the points view, times the output-equivalent factor', () => {
// Knots at iv 20/50/100 read exactly at those tiers: 1000/400/100.
const tiers = [20, 50, 100];
const feed = computeTcoFeed(threeKnotRows(), WORKLOAD_8K1K, tiers);
const scores = computeTcoScores(feed, WORKLOAD_8K1K, tiers, [0.5, 0.3, 0.2], [1], 0.25);
expect(scores).toHaveLength(1);
// 0.5·1000 + 0.3·400 + 0.2·100 = 640, ×(1 + 0.25·8192/1024) = ×3.
expect(scores[0].workload_scores['8192x1024']).toBe(640);
expect(scores[0].score).toBe(1920);
// alpha=0 scores plain output throughput.
const plain = computeTcoScores(feed, WORKLOAD_8K1K, tiers, [0.5, 0.3, 0.2], [1], 0);
expect(plain[0].score).toBe(640);
});

it('unreachable tiers contribute 0 at full weight; clamped tiers contribute the clamp', () => {
// Frontier spans iv 40 → 80.
const source = [makeRow({ itl: 1 / 40, otput: 800 }), makeRow({ itl: 1 / 80, otput: 200 })];
const tiers = [30, 40, 100];
const feed = computeTcoFeed(source, WORKLOAD_8K1K, tiers);
const scores = computeTcoScores(feed, WORKLOAD_8K1K, tiers, [0.25, 0.5, 0.25], [1], 0);
// 0.25·800 (clamped) + 0.5·800 + 0.25·0 (unreachable) = 600 — the
// unreachable tier's weight is NOT redistributed to reachable tiers.
expect(scores[0].score).toBe(600);
expect(scores[0].unreachable_tiers).toBe(1);
expect(scores[0].clamped_tiers).toBe(1);
});

it('blends workloads and renormalizes over covered ones for partial coverage', () => {
const feed = computeTcoFeed(twoWorkloadRows(), BOTH_WORKLOADS, [50]);
const scores = computeTcoScores(feed, BOTH_WORKLOADS, [50], [1], [0.5, 0.5], 0);
expect(scores.map((s) => s.hardware)).toEqual(['b200', 'gb300']);

const b200 = scores[0];
expect(b200.workload_scores).toEqual({ '1024x1024': 1000, '8192x1024': 400 });
expect(b200.score).toBe(700); // 0.5·1000 + 0.5·400
expect(b200.workloads_covered).toBe(2);

// gb300 lacks 1k1k → its weight renormalizes onto 8k1k alone.
const gb300 = scores[1];
expect(gb300.workload_scores).toEqual({ '1024x1024': null, '8192x1024': 5000 });
expect(gb300.score).toBe(5000);
expect(gb300.workloads_covered).toBe(1);
});

it('applies per-workload output-equivalent factors before blending', () => {
const feed = computeTcoFeed(twoWorkloadRows(), BOTH_WORKLOADS, [50]);
const scores = computeTcoScores(feed, BOTH_WORKLOADS, [50], [1], [0.5, 0.5], 0.25);
// 0.5·1000·(1 + 0.25·1) + 0.5·400·(1 + 0.25·8) = 625 + 600.
expect(scores[0].score).toBe(1225);
});

it('scores 0 when the only covered workloads carry zero weight', () => {
const feed = computeTcoFeed(twoWorkloadRows(), BOTH_WORKLOADS, [50]);
const scores = computeTcoScores(feed, BOTH_WORKLOADS, [50], [1], [1, 0], 0);
const gb300 = scores.find((s) => s.hardware === 'gb300')!;
expect(gb300.score).toBe(0);
});

it('carries the newest latest_date and oldest frontier date across covered workloads', () => {
const feed = computeTcoFeed(twoWorkloadRows(), BOTH_WORKLOADS, [50]);
const scores = computeTcoScores(feed, BOTH_WORKLOADS, [50], [1], [0.5, 0.5], 0);
expect(scores[0].latest_date).toBe('2026-07-01');
expect(scores[0].oldest_frontier_date).toBe('2026-05-01');
});

it('returns an empty list for an empty feed', () => {
expect(computeTcoScores([], BOTH_WORKLOADS, [50], [1], [0.5, 0.5], 0)).toEqual([]);
});
});

describe('tcoScoresToCsv', () => {
it('emits one score_<workload> column per requested workload, blank when uncovered', () => {
const feed = computeTcoFeed(twoWorkloadRows(), BOTH_WORKLOADS, [50]);
const scores = computeTcoScores(feed, BOTH_WORKLOADS, [50], [1], [0.5, 0.5], 0);
const csv = tcoScoresToCsv(scores, BOTH_WORKLOADS);
const lines = csv.split('\n');
expect(lines[0]).toBe(
'hardware,score,score_1024x1024,score_8192x1024,workloads_covered,' +
'unreachable_tiers,clamped_tiers,latest_date,oldest_frontier_date',
);
expect(lines[1]).toBe('b200,700,1000,400,2,0,0,2026-07-01,2026-05-01');
expect(lines[2]).toBe('gb300,5000,,5000,1,0,0,2026-07-10,2026-07-10');
expect(csv.endsWith('\n')).toBe(true);
});
});
Loading
Loading