From f614a0ec77f2e229d5dd8a6a252fa717ffe9c778 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:11:01 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(inference):=20derive=20$/M=20tok=20and?= =?UTF-8?q?=20J/token=20from=20throughput=20instead=20of=20splining=20them?= =?UTF-8?q?=20/=20=E6=AF=8F=20token=20=E6=88=90=E6=9C=AC=E4=B8=8E=E8=83=BD?= =?UTF-8?q?=E8=80=97=E6=94=B9=E4=B8=BA=E6=8C=89=E5=90=9E=E5=90=90=E9=87=8F?= =?UTF-8?q?=E6=8E=A8=E5=AF=BC=E8=80=8C=E9=9D=9E=E7=9B=B4=E6=8E=A5=E6=8F=92?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cost per million tokens and joules per token are a per-chip constant divided by a throughput. The interpolation splined them directly, which averages reciprocals; 1/x is convex, so the result diverged from the value implied by the interpolated throughput. /inference settles which is right. It plots these metrics only at measured points (lib/chart-utils.ts:380, roof: false), so its values are the oracle, and they satisfy `metric x throughput = constant` by construction. Holding out each interior frontier knot and predicting its real value: splined metric mean err 162.8% closer on 36 / 144 derived from throughput mean err 42.2% closer on 108 / 144 The splined pair also broke that identity by a median 71.6% and up to 2026% — reporting operating points no real config could occupy. The two agree exactly at all 470 measured knots; the divergence is entirely between them, and the splined read is the higher one 73.6% of the time (max 25.3x on sparsely swept frontiers, where adjacent knots differ ~100x in throughput). Changed: interpolateForGPU, maxInteractivityAtCost and interpolateMetricAtInteractivity now spline the throughput these metrics divide and re-derive. iso_interactivity.py is synced in this commit per AGENTS.md and verified to agree to the last digit (0.44517072882391184 on a shared fixture); that rule now also covers recoverReciprocalNumerator. recoverReciprocalNumerator returns null unless every usable point agrees on the constant, and all call sites then fall back to splining. That guard is load bearing: the measured* energy keys have a numerator measured per point, so they stay splined, and hand-built points whose cost is unrelated to their throughput keep their previous values instead of being silently rewritten. The rate is recovered across all three token types at once — checking one family and falling back to another recovers a rate from output tokens and applies it to total throughput, which the existing maxInteractivityAtCost tests caught. Published numbers move down, since they were the overstated side. Over 555 unclamped reads (dsr1 8k/1k, targets 20-75), two-thirds shift under 10%; the tail is the sparsely swept disaggregated configs, worst case a 25x drop. Note for whoever merges second: the Fleet Lifecycle branch documents this as an open follow-up and derives its break-even to work around it. That prose needs reconciling once both land — the workaround stays correct, it is just no longer a workaround. --- .../iso_interactivity.py | 74 ++++++++- AGENTS.md | 3 + docs/tco-calculator.md | 59 +++++++ .../components/calculator/interpolation.ts | 142 ++++++++++++++++- .../calculator/useThroughputData.test.ts | 148 ++++++++++++++++++ .../calculator/useThroughputData.ts | 4 + .../hooks/useInterpolatedTrendData.ts | 51 ++++++ 7 files changed, 473 insertions(+), 8 deletions(-) diff --git a/.claude/skills/write-inferencex-blog/iso_interactivity.py b/.claude/skills/write-inferencex-blog/iso_interactivity.py index ab38e6e61..1fcbd1ded 100644 --- a/.claude/skills/write-inferencex-blog/iso_interactivity.py +++ b/.claude/skills/write-inferencex-blog/iso_interactivity.py @@ -18,11 +18,23 @@ 6. Clamp the interpolated metric value to [min(ys), max(ys)] of the frontier to prevent cubic-spline overshoots above/below the data. +Reciprocal metrics ($/M tok, J/token) are a special case: they are a per-chip +constant divided by a throughput, so they are NOT splined directly. Splining +them averages reciprocals, and 1/x is convex, so the result diverges from the +value implied by the interpolated throughput — by (1+r)^2/(4r) for a knot pair +whose throughputs differ by r, reaching 25x on sparsely swept frontiers (higher +73.6% of the time on real data; Steffen slopes can undershoot the chord). Pass +`reciprocal_of='throughput'` (or the matching output/input throughput key) to +spline that throughput and re-derive the metric, which is what the dashboard +does. See docs/tco-calculator.md. + Usage as a module: from iso_interactivity import interpolate_metric, pareto_front_upper_left # points: list of dicts with at least 'interactivity' and 'throughput', # plus whatever metric you want to interpolate. - cost = interpolate_metric(points, target_iv=18.0, metric_key='cost_per_M') + # cost/energy are reciprocal in throughput — name the throughput they divide + cost = interpolate_metric(points, target_iv=18.0, metric_key='cost_per_M', + reciprocal_of='throughput') Usage as a script (JSON input on stdin, JSON output on stdout): echo '{"points":[...], "target_iv":18.0, "metric_key":"throughput"}' \\ @@ -143,12 +155,43 @@ def hermite_interpolate( return h00 * ys[lo] + h10 * hh * m[lo] + h01 * ys[hi] + h11 * hh * m[hi] +def recover_reciprocal_numerator( + values: list[float], throughputs: list[float] +) -> Optional[float]: + """Recover the constant `c` of a metric defined as `c / throughput`. + + 1:1 with `recoverReciprocalNumerator` in + packages/app/src/components/calculator/interpolation.ts. Returns None unless + EVERY usable point agrees on the constant: the identity is what licenses + re-deriving the metric, so data whose numerator actually varies per point + must fall back to being splined directly. + """ + # 0.1%, matching the TS side. Deliberately loose: blog tables are assembled + # from costs written to a few decimals, and a tight gate would silently + # reject them and fall back to splining. Still rejects a numerator that + # genuinely varies per point (measured power moves by whole percent). + relative_tolerance = 1e-3 + numerator: Optional[float] = None + for value, throughput in zip(values, throughputs): + if value is None or throughput is None: + continue + if not value > 0 or not throughput > 0: + continue + candidate = value * throughput + if numerator is None: + numerator = candidate + elif abs(candidate - numerator) > abs(numerator) * relative_tolerance: + return None + return numerator + + def interpolate_metric( points: list[dict], target_iv: float, metric_key: str = 'throughput', iv_key: str = 'interactivity', tput_key: str = 'throughput', + reciprocal_of: Optional[str] = None, ) -> Optional[float]: """Interpolate `metric_key` at `target_iv` using the chart's algorithm. @@ -160,6 +203,12 @@ def interpolate_metric( of which metric you're interpolating — this matches the chart, where the frontier is defined by the upper-left throughput envelope and other metrics (cost, energy, TPOT, ...) are derived values at frontier points. + + `reciprocal_of` names the throughput key a metric divides, for metrics of the + form `constant / throughput` ($/M tok, J/token). Set it for those metrics: + that throughput is splined and the metric re-derived, matching the dashboard. + Leave it None for metrics splined directly (throughput itself, tok/s/MW, + latency, and measured energy — whose numerator varies per point). """ if not points: return None @@ -189,6 +238,25 @@ def interpolate_metric( # point falls back to 0 instead of raising KeyError. Use `.get` so the # CLI returns null cleanly instead of dying with a traceback. ys = [(p.get(metric_key) if p.get(metric_key) is not None else 0) for p in sorted_front] + + if reciprocal_of is not None: + tputs = [ + (p.get(reciprocal_of) if p.get(reciprocal_of) is not None else 0) + for p in sorted_front + ] + numerator = recover_reciprocal_numerator(ys, tputs) + # None means these points do not obey the identity — fall through and + # spline the metric directly, matching the TS fallback. + if numerator is not None: + tput_slopes = monotone_slopes(xs, tputs) + tput = hermite_interpolate(xs, tputs, tput_slopes, target_iv) + # Clamp the throughput, as the TS side does, then derive; cost then + # cannot overshoot the frontier's own cost range either. + tput = max(min(tputs), min(max(tputs), tput)) + if tput <= 0: + return 0.0 + return numerator / tput + slopes = monotone_slopes(xs, ys) raw = hermite_interpolate(xs, ys, slopes, target_iv) @@ -203,7 +271,8 @@ def interpolate_metric( def _cli() -> None: - """Stdin: {"points": [...], "target_iv": N, "metric_key": "..."} + """Stdin: {"points": [...], "target_iv": N, "metric_key": "...", + "reciprocal_of": "throughput" for $/M tok and J/token} Stdout: {"value": N or null}""" req = json.loads(sys.stdin.read()) value = interpolate_metric( @@ -212,6 +281,7 @@ def _cli() -> None: metric_key=req.get('metric_key', 'throughput'), iv_key=req.get('iv_key', 'interactivity'), tput_key=req.get('tput_key', 'throughput'), + reciprocal_of=req.get('reciprocal_of'), ) json.dump({'value': value}, sys.stdout) sys.stdout.write('\n') diff --git a/AGENTS.md b/AGENTS.md index 645d0bd7b..ebd225e97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,11 +193,14 @@ The Python helper is a 1:1 port of these three TypeScript functions: Plus the wrapper `interpolateMetricAtInteractivity` in `packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts` which composes them with the "no extrapolation → return null" rule. +Plus `recoverReciprocalNumerator` in `interpolation.ts`, which decides whether a metric is splined directly or derived from the interpolated throughput. $/M tok and J/token are a per-chip constant over a throughput, so splining them averages reciprocals and overstates the value between knots; both TS and Python spline that throughput and re-derive instead. See `docs/tco-calculator.md` for the measurements. + **Rule: any PR that changes any of those four TypeScript functions MUST also update `.claude/skills/write-inferencex-blog/iso_interactivity.py` in the same commit.** Drift between the TS and Python implementations means the blog tables will silently diverge from the live chart on the very next post — readers will see one number in the table and a different one in the chart they click through to. This includes: - Changing the Pareto frontier definition (upper-left → lower-left, or adding tie-breaking rules) - Switching from Steffen's monotone slopes to a different spline construction (Fritsch-Carlson, natural cubic, etc.) - Loosening or tightening the extrapolation rule (currently: return `null` outside `[min x, max x]`) +- Changing which metrics are derived from throughput rather than splined, or the tolerance that decides whether the data obeys `metric x throughput = constant` - Adjusting the Y-clamp behavior that prevents spline overshoot The Python file has a header comment explaining the pipeline and a `_cli()` entrypoint for stdin/stdout JSON usage. When you update it, keep the structure 1:1 with the TS so future readers can diff the two files line by line. Run the helper against a known dataset and confirm the outputs match what the chart renders before merging. diff --git a/docs/tco-calculator.md b/docs/tco-calculator.md index ade7149ef..9f5ebf87f 100644 --- a/docs/tco-calculator.md +++ b/docs/tco-calculator.md @@ -121,3 +121,62 @@ Agentic interactivity follows the same definition as the main inference chart: `b300Rows` in `cypress/support/overlay-fixtures.ts` covers the agentic calculator path; `singleTurnRows` remains the fixture for fixed-sequence visibility and sequence-switching behavior. + +## Reciprocal Metrics Are Derived, Not Splined + +`$/M tok` and `J/token` are a per-chip constant divided by a throughput +(`$/GPU-hr x 1e6 / (tok/s x 3600)`, `W / (tok/s)`). `interpolateForGPU`, +`maxInteractivityAtCost` and `interpolateMetricAtInteractivity` therefore spline +the **throughput** those metrics divide and re-derive the metric, rather than +splining the metric itself. + +Splining them directly averages reciprocals. Because `1/x` is convex, the result +sits away from the value implied by the interpolated throughput — by +`(1+r)^2/(4r)` for a knot pair whose throughputs differ by `r`, which reaches +~25x on sparsely swept frontiers (`r` up to 103). Measured over the captured +fixture: the two agree exactly at all 470 frontier knots, and between knots the +splined read is higher 73.6% of the time (median 1.057, p95 3.97, max 25.3, min +0.95 — the direction is usual, not universal, because Steffen slopes make the +cubic undershoot the chord in some brackets). + +`/inference` is the tiebreak: it plots these metrics only at measured points +(`lib/chart-utils.ts:380`, `roof: false`), so its values are the oracle and they +satisfy `metric x throughput = constant` by construction. Leave-one-out over 144 +held-out interior knots: + +| method | mean err | p50 | p90 | p99 | closer to oracle | +| ----------------------- | -------- | ----- | ------ | ------- | ---------------- | +| splined metric | 162.8% | 86.3% | 528.1% | 1402.1% | 36 / 144 | +| derived from throughput | 42.2% | 23.0% | 72.0% | 245.3% | **108 / 144** | + +Deriving is ~4x closer and preserves the identity; splining broke it by a median +71.6% and up to 2026%, reporting operating points no real config could occupy. + +### The consistency guard + +`recoverReciprocalNumerator` returns the constant only if **every** usable point +agrees on it (1e-9 relative). That guard is what licenses the rewrite, and it is +not theoretical: the `measured*` energy keys have a numerator measured per point +rather than a constant, so they are excluded from `RECIPROCAL_OF_THROUGHPUT` and +still splined directly. When the guard fails, all three call sites fall back to +splining — so synthetic or hand-built points whose cost is unrelated to their +throughput keep their previous behaviour instead of being silently rewritten. + +The rate is recovered across **all three token types at once** (`recoverCostRate`). +Checking one family alone and falling back to another would recover a rate from +output tokens and then apply it to total throughput; the existing +`maxInteractivityAtCost` tests caught exactly that mistake. + +### Impact on published numbers + +Costs come down, since they were the overstated side. Over 555 unclamped reads +(dsr1 8k/1k, targets 20-75), new/old: + +| metric | p05 | p50 | p95 | min | within 1% | within 10% | +| -------------- | ----- | ----- | ----- | ----- | --------- | ---------- | +| $/M tok total | 0.379 | 0.963 | 1.026 | 0.039 | 19.1% | 66.5% | +| $/M tok output | 0.220 | 0.948 | 1.026 | 0.020 | 17.8% | 61.3% | +| $/M tok input | 0.553 | 0.976 | 1.026 | 0.100 | 27.6% | 75.0% | + +Two-thirds of reads move under 10%; the tail is the sparsely swept disaggregated +configs, which is also where the old numbers were least defensible. diff --git a/packages/app/src/components/calculator/interpolation.ts b/packages/app/src/components/calculator/interpolation.ts index 159d0060f..d2b3abee9 100644 --- a/packages/app/src/components/calculator/interpolation.ts +++ b/packages/app/src/components/calculator/interpolation.ts @@ -158,6 +158,99 @@ export function getCostField( return p[costProvider]; } +/** + * Recover the constant numerator `c` of a metric that is defined as + * `c / throughput`, from any frontier point that carries both values. + * + * Cost per million tokens is `$/GPU-hr x 1e6 / (tok/s x 3600)` and energy per + * token is `W / (tok/s)`: both are a per-chip constant divided by a throughput, + * and the constant is identical at every point of a config. Recovering it from + * the points avoids threading the hardware registry into this module, which must + * stay dependency-free for the Python port. + * + * Why this exists: splining such a metric directly is wrong. It averages + * reciprocals, and `1/x` is convex, so the interpolated metric diverges from the + * value implied by the interpolated throughput — by `(1+r)^2/(4r)` for a knot + * pair whose throughputs differ by `r`, and `r` reaches ~100 on sparsely swept + * frontiers. Measured on real frontiers, the splined read is the higher one + * 73.6% of the time (median 1.057, max 25.3, min 0.95 — Steffen slopes let the + * cubic undershoot the chord in some brackets, so the direction is usual rather + * than universal). Deriving from the interpolated throughput preserves the + * identity `metric x throughput = c` that the per-point values on /inference obey + * by construction, and lands ~4x closer to held-out measured points. See + * docs/tco-calculator.md. + * + * Returns null unless EVERY usable point agrees on the constant. That check is + * the safety rail: the identity is what licenses re-deriving the metric, so a + * metric whose numerator actually varies per point (measured power, say) must + * fall back to being splined directly rather than have its values silently + * rewritten from one point's ratio. + */ +export function recoverReciprocalNumerator( + values: readonly number[], + throughputs: readonly number[], +): number | null { + // Dashboard values come from one getGpuSpecs(hwKey) lookup, so they agree to + // float rounding (~1e-16). The tolerance is far looser than that on purpose: + // the Python port is fed hand-assembled JSON for blog tables, where costs are + // written to a few decimals and a 1e-9 gate would silently reject them and + // fall back to the worse method. 0.1% still comfortably rejects what this + // guard is for — a numerator that genuinely varies per point, like measured + // power, which moves by whole percent across a sweep. And a numerator varying + // by less than 0.1% makes deriving and splining agree anyway. + const RELATIVE_TOLERANCE = 1e-3; + let numerator: number | null = null; + + for (let i = 0; i < values.length; i += 1) { + const value = values[i]; + const throughput = throughputs[i]; + if (value === undefined || throughput === undefined) continue; + if (!(value > 0) || !(throughput > 0)) continue; + + const candidate = value * throughput; + if (numerator === null) { + numerator = candidate; + } else if (Math.abs(candidate - numerator) > Math.abs(numerator) * RELATIVE_TOLERANCE) { + return null; + } + } + + return numerator; +} + +/** Evaluate a `numerator / throughput` metric at an interpolated throughput. */ +export function reciprocalMetricAt(numerator: number | null, throughput: number): number { + if (numerator === null || !(throughput > 0)) return 0; + return numerator / throughput; +} + +/** + * The single provider rate that must explain every cost field on these points, + * or null if it does not. + * + * All three token types share one `$/GPU-hr x 1e6/3600` and differ only in the + * throughput they divide, so consistency has to be checked across all of them + * together. Checking one family in isolation and falling back to another would + * recover a rate from output tokens and then apply it to total throughput. + */ +function recoverCostRate( + sorted: readonly GPUDataPoint[], + costProvider: CostProvider, +): number | null { + return recoverReciprocalNumerator( + [ + ...sorted.map((p) => getCostField(p, costProvider, 'total')), + ...sorted.map((p) => getCostField(p, costProvider, 'output')), + ...sorted.map((p) => getCostField(p, costProvider, 'input')), + ], + [ + ...sorted.map((p) => p.throughput), + ...sorted.map((p) => p.outputThroughput), + ...sorted.map((p) => p.inputThroughput), + ], + ); +} + /** * Given a set of data points for a single GPU, find the maximum interactivity * (tok/s/user) whose interpolated cost per million tokens stays at or below @@ -195,12 +288,30 @@ export function maxInteractivityAtCost( } const xs = sorted.map((p) => p.interactivity); - const ys = sorted.map(getCost); + // Cost is derived from the interpolated throughput of the selected token type, + // exactly as interpolateForGPU does it — otherwise this inverse lookup would + // answer against a different cost curve than the bars the user is reading. + const getTput = (p: GPUDataPoint) => + costType === 'input' + ? p.inputThroughput + : costType === 'output' + ? p.outputThroughput + : p.throughput; + const tputs = sorted.map(getTput); + const costs = sorted.map(getCost); + // Same decision as interpolateForGPU, so the inverse lookup and the bars can + // never answer against different cost curves. + const rate = recoverCostRate(sorted, costProvider); + // Spline whichever series the identity licenses: the throughput when cost is + // genuinely `rate / throughput`, otherwise cost itself (previous behaviour). + const ys = rate === null ? costs : tputs; const slopes = monotoneSlopes(xs, ys); - // Same overshoot clamp as interpolateForGPU's buildMetric + // Same overshoot clamp as interpolateForGPU's buildMetric. const lo = Math.min(...ys); const hi = Math.max(...ys); - const costAt = (x: number) => Math.max(lo, Math.min(hi, hermiteInterpolate(xs, ys, slopes, x))); + const splineAt = (x: number) => Math.max(lo, Math.min(hi, hermiteInterpolate(xs, ys, slopes, x))); + const costAt = (x: number) => + rate === null ? splineAt(x) : reciprocalMetricAt(rate, splineAt(x)); const minX = xs[0]; const maxX = xs.at(-1)!; @@ -315,9 +426,28 @@ export function interpolateForGPU( const value = buildMetric(getOutputValue); const outputTputValue = buildMetric((p) => p.outputThroughput); const inputTputValue = buildMetric((p) => p.inputThroughput); - const cost = buildMetric((p) => getCostField(p, costProvider, 'total')); - const costInput = buildMetric((p) => getCostField(p, costProvider, 'input')); - const costOutput = buildMetric((p) => getCostField(p, costProvider, 'output')); + + // Cost is `$/GPU-hr / tokens` — a constant over a throughput — so it is + // derived from the interpolated throughput rather than splined itself. See + // `recoverReciprocalNumerator`. In throughput_to_interactivity mode the target + // axis *is* total throughput, so the clamped target is the value to divide by. + const totalTputAtTarget = mode === 'interactivity_to_throughput' ? value : clampedTarget; + const rate = recoverCostRate(sorted, costProvider); + // `rate === null` means these points do not obey the identity, so the metric + // is splined directly as before rather than rewritten from one point's ratio. + const cost = + rate === null + ? buildMetric((p) => getCostField(p, costProvider, 'total')) + : reciprocalMetricAt(rate, totalTputAtTarget); + const costInput = + rate === null + ? buildMetric((p) => getCostField(p, costProvider, 'input')) + : reciprocalMetricAt(rate, inputTputValue); + const costOutput = + rate === null + ? buildMetric((p) => getCostField(p, costProvider, 'output')) + : reciprocalMetricAt(rate, outputTputValue); + const tpPerMw = buildMetric((p) => p.tpPerMw); const inputTpPerMw = buildMetric((p) => p.inputTpPerMw); const outputTpPerMw = buildMetric((p) => p.outputTpPerMw); diff --git a/packages/app/src/components/calculator/useThroughputData.test.ts b/packages/app/src/components/calculator/useThroughputData.test.ts index a9bfd05c4..e439ef6e2 100644 --- a/packages/app/src/components/calculator/useThroughputData.test.ts +++ b/packages/app/src/components/calculator/useThroughputData.test.ts @@ -13,6 +13,7 @@ import { maxInteractivityAtCost, monotoneSlopes, paretoFrontUpperLeft, + recoverReciprocalNumerator, sign, } from './useThroughputData'; @@ -1315,3 +1316,150 @@ describe('interpolateForGPU clamped flag', () => { ); }); }); + +// ========================================================================= +// Reciprocal metrics — cost is derived from throughput, not splined +// ========================================================================= + +describe('recoverReciprocalNumerator', () => { + it('recovers the constant when every point agrees', () => { + expect(recoverReciprocalNumerator([1, 2, 4], [400, 200, 100])).toBe(400); + }); + + it('tolerates float rounding, which is all production data shows', () => { + const k = 516.6666666666666; + const tputs = [1234.5, 987.65, 321.098]; + const values = tputs.map((t) => k / t); + expect(recoverReciprocalNumerator(values, tputs)).toBeCloseTo(k, 9); + }); + + it('returns null when the points disagree — the numerator is not constant', () => { + // This is the safety rail: a metric whose numerator varies per point (e.g. + // measured power) must not have its values rewritten from one point's ratio. + expect(recoverReciprocalNumerator([1, 2], [400, 300])).toBeNull(); + }); + + it('accepts costs rounded for publication but rejects a percent-level drift', () => { + // The tolerance is 0.1%. Blog tables are hand-assembled from costs written + // to a few decimals, and iso_interactivity.py must not silently fall back on + // them; a numerator that genuinely varies moves far more than this. + expect( + recoverReciprocalNumerator([0.0964, 0.2892, 0.6427, 1.928], [6000, 2000, 900, 300]), + ).not.toBeNull(); + // 1% apart — rejected. + expect(recoverReciprocalNumerator([1, 2.02], [400, 200])).toBeNull(); + }); + + it('skips unusable pairs rather than treating them as disagreement', () => { + expect(recoverReciprocalNumerator([0, 1, 2], [400, 400, 200])).toBe(400); + expect(recoverReciprocalNumerator([1, 2], [0, 200])).toBe(400); + }); + + it('returns null when no pair is usable', () => { + expect(recoverReciprocalNumerator([0, 0], [100, 200])).toBeNull(); + expect(recoverReciprocalNumerator([], [])).toBeNull(); + }); +}); + +describe('interpolateForGPU cost derivation', () => { + /** Points obeying the real identity: cost = rate / throughput. */ + const RATE = 578.4; // $/GPU-hr x 1e6 / 3600 + const consistent = (interactivity: number, throughput: number) => + makePoint({ + interactivity, + throughput, + outputThroughput: throughput, + inputThroughput: throughput, + costh: RATE / throughput, + costhOutput: RATE / throughput, + costhi: RATE / throughput, + }); + + const frontier = [ + consistent(10, 6000), + consistent(30, 2000), + consistent(55, 900), + consistent(80, 300), + ]; + + it('keeps cost x throughput equal to the provider rate between measured points', () => { + // The identity /inference's per-point values obey by construction. Splining + // cost independently breaks it by up to 25x on sparse frontiers. + for (const target of [15, 20, 42, 60, 75]) { + const r = interpolateForGPU(frontier, target, 'interactivity_to_throughput', 'costh')!; + expect(r.cost * r.value).toBeCloseTo(RATE, 6); + expect(r.costOutput * r.outputTputValue).toBeCloseTo(RATE, 6); + expect(r.costInput * r.inputTputValue).toBeCloseTo(RATE, 6); + } + }); + + it('matches the Python port bundled with the blog skill', () => { + // iso_interactivity.py, same frontier and target, reciprocal_of=throughput. + // Drift here means blog tables diverge from the live chart — see AGENTS.md. + const r = interpolateForGPU(frontier, 42, 'interactivity_to_throughput', 'costh')!; + expect(r.cost).toBeCloseTo(0.44517072882391184, 12); + }); + + it('is exact at a measured point, where both methods agree anyway', () => { + const r = interpolateForGPU(frontier, 30, 'interactivity_to_throughput', 'costh')!; + expect(r.value).toBeCloseTo(2000, 6); + expect(r.cost).toBeCloseTo(RATE / 2000, 9); + }); + + it('reads below the old splined value on a two-knot bracket, near (1+r)^2/(4r)', () => { + // The mechanism: splining cost averages reciprocals (arithmetic mean) while + // deriving takes the reciprocal of an average (harmonic mean), and + // AM/HM = (1+r)^2/(4r) for throughputs differing by r. Steffen slopes make + // even a two-knot spline a slight cubic rather than a chord, so the match is + // close but not exact. Beyond two knots the cubic can undershoot the chord, + // so the direction is usual but not universal — measured at 73.6% high over + // real frontiers, min 0.95. See docs/tco-calculator.md. + const t1 = 6000; + const t2 = 300; + const xs = [10, 80]; + const pair = [consistent(xs[0]!, t1), consistent(xs[1]!, t2)]; + const mid = (xs[0]! + xs[1]!) / 2; + + const derived = interpolateForGPU(pair, mid, 'interactivity_to_throughput', 'costh')!.cost; + // Reconstruct the previous behaviour: spline the cost values themselves. + const costs = [RATE / t1, RATE / t2]; + const splined = hermiteInterpolate(xs, costs, monotoneSlopes(xs, costs), mid); + + expect(derived).toBeLessThan(splined); + const r = t1 / t2; + expect(splined / derived).toBeCloseTo((1 + r) ** 2 / (4 * r), 0); + // and the derived value still satisfies the identity, which splining does not + const at = interpolateForGPU(pair, mid, 'interactivity_to_throughput', 'costh')!; + expect(at.cost * at.value).toBeCloseTo(RATE, 6); + }); + + it('falls back to splining when the points do not obey the identity', () => { + // Synthetic points whose cost is unrelated to throughput keep the old + // behaviour rather than being silently rewritten. + const inconsistent = [ + makePoint({ interactivity: 10, throughput: 1000, costh: 0.2 }), + makePoint({ interactivity: 40, throughput: 200, costh: 2 }), + ]; + const r = interpolateForGPU(inconsistent, 25, 'interactivity_to_throughput', 'costh')!; + expect(r.cost).toBeGreaterThan(0.2); + expect(r.cost).toBeLessThan(2); + // Not the derived value, which would have been 200/interpolated-throughput. + expect(r.cost * r.value).not.toBeCloseTo(200, 3); + }); + + it('derives cost from the target axis in throughput_to_interactivity mode', () => { + const r = interpolateForGPU(frontier, 1500, 'throughput_to_interactivity', 'costh')!; + // In reverse mode the target IS total throughput, so cost follows from it. + expect(r.cost).toBeCloseTo(RATE / 1500, 9); + }); + + it('agrees with maxInteractivityAtCost on the derived curve', () => { + const budget = 0.45; + const iv = maxInteractivityAtCost(frontier, budget, 'costh', 'total')!; + expect(iv).not.toBeNull(); + const at = interpolateForGPU(frontier, iv, 'interactivity_to_throughput', 'costh')!; + expect(at.cost).toBeLessThanOrEqual(budget + 1e-6); + const above = interpolateForGPU(frontier, iv + 1, 'interactivity_to_throughput', 'costh')!; + expect(above.cost).toBeGreaterThan(budget); + }); +}); diff --git a/packages/app/src/components/calculator/useThroughputData.ts b/packages/app/src/components/calculator/useThroughputData.ts index f8ad7da45..fe2c25493 100644 --- a/packages/app/src/components/calculator/useThroughputData.ts +++ b/packages/app/src/components/calculator/useThroughputData.ts @@ -20,6 +20,8 @@ import { maxInteractivityAtCost, monotoneSlopes, paretoFrontUpperLeft, + reciprocalMetricAt, + recoverReciprocalNumerator, sign, } from './interpolation'; import { restrictAgenticPointsToE2eFrontier } from '@/lib/agentic-frontier'; @@ -33,6 +35,8 @@ export { maxInteractivityAtCost, monotoneSlopes, paretoFrontUpperLeft, + reciprocalMetricAt, + recoverReciprocalNumerator, sign, }; diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index cde86370d..78b0a3439 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -7,6 +7,8 @@ import { hermiteInterpolate, monotoneSlopes, paretoFrontUpperLeft, + recoverReciprocalNumerator, + reciprocalMetricAt, } from '@/components/calculator/useThroughputData'; import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history'; import { getHardwareKey } from '@/lib/chart-utils'; @@ -93,6 +95,34 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { return point; } +/** + * Metrics defined as a per-chip constant divided by a throughput, mapped to the + * throughput metric they divide. These are interpolated by splining that + * throughput and re-deriving, never by splining the metric itself — `1/x` is + * convex, so splining it overstates the value between measured points. See + * `recoverReciprocalNumerator` and docs/tco-calculator.md. + * + * The `measured*` energy keys are deliberately absent: their numerator is + * measured per point rather than a constant, so the identity does not hold and + * splining them directly remains correct. + */ +const RECIPROCAL_OF_THROUGHPUT: Partial> = { + // $/M tok = $/GPU-hr x 1e6 / (tok/s x 3600) + costh: 'tpPerGpu', + costn: 'tpPerGpu', + costr: 'tpPerGpu', + costhOutput: 'outputTputPerGpu', + costnOutput: 'outputTputPerGpu', + costrOutput: 'outputTputPerGpu', + costhi: 'inputTputPerGpu', + costni: 'inputTputPerGpu', + costri: 'inputTputPerGpu', + // J/token = W / (tok/s) + jTotal: 'tpPerGpu', + jOutput: 'outputTputPerGpu', + jInput: 'inputTputPerGpu', +}; + /** * Interpolate a selected metric at a target interactivity for a set of InferenceData points * from a single GPU. Uses Pareto front (throughput-based frontier) + monotone cubic Hermite spline. @@ -141,6 +171,27 @@ export function interpolateMetricAtInteractivity( metricYs.push(v); } + // Cost and energy per token are `constant / throughput`. Spline that + // throughput and re-derive rather than splining the metric, so the value + // agrees with the per-point figures on the inference chart. + const throughputKey = RECIPROCAL_OF_THROUGHPUT[metricKey]; + if (throughputKey) { + const tputYs: number[] = []; + for (const p of sorted) { + const v = extractMetric(p, throughputKey); + if (v === null) return null; + tputYs.push(v); + } + const numerator = recoverReciprocalNumerator(metricYs, tputYs); + // null means these points do not obey the identity — fall through and spline + // the metric directly rather than rewrite it from one point's ratio. + if (numerator !== null) { + const tputSlopes = monotoneSlopes(xs, tputYs); + const tput = hermiteInterpolate(xs, tputYs, tputSlopes, targetInteractivity); + return reciprocalMetricAt(numerator, Math.max(0, tput)); + } + } + // Monotone cubic Hermite spline interpolation const slopes = monotoneSlopes(xs, metricYs); const interpolated = hermiteInterpolate(xs, metricYs, slopes, targetInteractivity); From 4138124464d1f5ed60b09d0b73be08e0713ac18a Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:59:02 +0800 Subject: [PATCH 2/2] fix(interpolation): align reciprocal metric validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the bundled Python helper from the existing unit test, return null for missing reciprocal inputs, and remove snapshot-specific impact claims that do not describe the changing live dataset. Correct the documented consistency tolerance to 0.1%. 中文:对齐倒数指标的插值验证。现有单元测试会直接运行随附的 Python helper;缺少倒数吞吐量输入时返回 null;移除无法代表持续变化的线上数据集的快照影响数字,并将一致性容差文档更正为 0.1%。 --- .../iso_interactivity.py | 38 +++++++------ AGENTS.md | 2 +- docs/tco-calculator.md | 56 +++++------------- .../components/calculator/interpolation.ts | 17 ++---- .../calculator/useThroughputData.test.ts | 57 +++++++++++++++++-- .../hooks/useInterpolatedTrendData.ts | 4 +- 6 files changed, 97 insertions(+), 77 deletions(-) diff --git a/.claude/skills/write-inferencex-blog/iso_interactivity.py b/.claude/skills/write-inferencex-blog/iso_interactivity.py index 1fcbd1ded..64058e8ba 100644 --- a/.claude/skills/write-inferencex-blog/iso_interactivity.py +++ b/.claude/skills/write-inferencex-blog/iso_interactivity.py @@ -19,14 +19,11 @@ frontier to prevent cubic-spline overshoots above/below the data. Reciprocal metrics ($/M tok, J/token) are a special case: they are a per-chip -constant divided by a throughput, so they are NOT splined directly. Splining -them averages reciprocals, and 1/x is convex, so the result diverges from the -value implied by the interpolated throughput — by (1+r)^2/(4r) for a knot pair -whose throughputs differ by r, reaching 25x on sparsely swept frontiers (higher -73.6% of the time on real data; Steffen slopes can undershoot the chord). Pass -`reciprocal_of='throughput'` (or the matching output/input throughput key) to -spline that throughput and re-derive the metric, which is what the dashboard -does. See docs/tco-calculator.md. +constant divided by a throughput, so they are NOT splined independently. Two +independent splines need not preserve `metric * throughput = constant` between +measured knots. Pass `reciprocal_of='throughput'` (or the matching output/input +throughput key) to spline that throughput and re-derive the metric, which is what +the dashboard does. See docs/tco-calculator.md for a reproducible measurement. Usage as a module: from iso_interactivity import interpolate_metric, pareto_front_upper_left @@ -156,7 +153,7 @@ def hermite_interpolate( def recover_reciprocal_numerator( - values: list[float], throughputs: list[float] + values: list[Optional[float]], throughputs: list[Optional[float]] ) -> Optional[float]: """Recover the constant `c` of a metric defined as `c / throughput`. @@ -234,16 +231,23 @@ def interpolate_metric( return sorted_front[0].get(metric_key) return None - # Matches TS `extractMetric(...) ?? 0`: missing metric on a frontier - # point falls back to 0 instead of raising KeyError. Use `.get` so the - # CLI returns null cleanly instead of dying with a traceback. - ys = [(p.get(metric_key) if p.get(metric_key) is not None else 0) for p in sorted_front] + # The TS helper returns null if any frontier point lacks the requested + # metric. Do the same here: coercing a missing value to zero would create a + # synthetic knot and make blog output diverge from the dashboard. + ys: list[float] = [] + for point in sorted_front: + value = point.get(metric_key) + if value is None: + return None + ys.append(value) if reciprocal_of is not None: - tputs = [ - (p.get(reciprocal_of) if p.get(reciprocal_of) is not None else 0) - for p in sorted_front - ] + tputs: list[float] = [] + for point in sorted_front: + throughput = point.get(reciprocal_of) + if throughput is None: + return None + tputs.append(throughput) numerator = recover_reciprocal_numerator(ys, tputs) # None means these points do not obey the identity — fall through and # spline the metric directly, matching the TS fallback. diff --git a/AGENTS.md b/AGENTS.md index ebd225e97..a50342f68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,7 +193,7 @@ The Python helper is a 1:1 port of these three TypeScript functions: Plus the wrapper `interpolateMetricAtInteractivity` in `packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts` which composes them with the "no extrapolation → return null" rule. -Plus `recoverReciprocalNumerator` in `interpolation.ts`, which decides whether a metric is splined directly or derived from the interpolated throughput. $/M tok and J/token are a per-chip constant over a throughput, so splining them averages reciprocals and overstates the value between knots; both TS and Python spline that throughput and re-derive instead. See `docs/tco-calculator.md` for the measurements. +Plus `recoverReciprocalNumerator` in `interpolation.ts`, which decides whether a metric is splined directly or derived from the interpolated throughput. $/M tok and J/token are a per-chip constant over a throughput, so independently splining the metric breaks that identity between knots; both TS and Python spline the throughput and re-derive instead. See `docs/tco-calculator.md` for the reproducible measurement. **Rule: any PR that changes any of those four TypeScript functions MUST also update `.claude/skills/write-inferencex-blog/iso_interactivity.py` in the same commit.** Drift between the TS and Python implementations means the blog tables will silently diverge from the live chart on the very next post — readers will see one number in the table and a different one in the chart they click through to. This includes: diff --git a/docs/tco-calculator.md b/docs/tco-calculator.md index 9f5ebf87f..2002753d3 100644 --- a/docs/tco-calculator.md +++ b/docs/tco-calculator.md @@ -130,53 +130,27 @@ sequence-switching behavior. the **throughput** those metrics divide and re-derive the metric, rather than splining the metric itself. -Splining them directly averages reciprocals. Because `1/x` is convex, the result -sits away from the value implied by the interpolated throughput — by -`(1+r)^2/(4r)` for a knot pair whose throughputs differ by `r`, which reaches -~25x on sparsely swept frontiers (`r` up to 103). Measured over the captured -fixture: the two agree exactly at all 470 frontier knots, and between knots the -splined read is higher 73.6% of the time (median 1.057, p95 3.97, max 25.3, min -0.95 — the direction is usual, not universal, because Steffen slopes make the -cubic undershoot the chord in some brackets). - -`/inference` is the tiebreak: it plots these metrics only at measured points -(`lib/chart-utils.ts:380`, `roof: false`), so its values are the oracle and they -satisfy `metric x throughput = constant` by construction. Leave-one-out over 144 -held-out interior knots: - -| method | mean err | p50 | p90 | p99 | closer to oracle | -| ----------------------- | -------- | ----- | ------ | ------- | ---------------- | -| splined metric | 162.8% | 86.3% | 528.1% | 1402.1% | 36 / 144 | -| derived from throughput | 42.2% | 23.0% | 72.0% | 245.3% | **108 / 144** | - -Deriving is ~4x closer and preserves the identity; splining broke it by a median -71.6% and up to 2026%, reporting operating points no real config could occupy. +Independently splining the reciprocal metric and throughput creates two curves +that need not satisfy `metric x throughput = constant` between measured knots. +The direction and size of the difference depend on frontier density and can +change as benchmark runs land. Re-deriving the metric preserves its definition +at every interpolated point. + +`/inference` plots these metrics only at measured points (`lib/chart-utils.ts`, +`roof: false`), where both methods agree exactly. Leave-one-out measurements can +compare interpolation models on a fixed snapshot, but they must not be presented +as permanent impact figures for the changing live dataset. ### The consistency guard `recoverReciprocalNumerator` returns the constant only if **every** usable point -agrees on it (1e-9 relative). That guard is what licenses the rewrite, and it is -not theoretical: the `measured*` energy keys have a numerator measured per point -rather than a constant, so they are excluded from `RECIPROCAL_OF_THROUGHPUT` and -still splined directly. When the guard fails, all three call sites fall back to -splining — so synthetic or hand-built points whose cost is unrelated to their -throughput keep their previous behaviour instead of being silently rewritten. +agrees on it within 0.1% (`1e-3` relative). That guard is what licenses the +rewrite. The `measured*` energy keys have a numerator measured per point rather +than a constant, so they are excluded from `RECIPROCAL_OF_THROUGHPUT` and still +splined directly. When the guard fails, all three call sites fall back to +splining. The rate is recovered across **all three token types at once** (`recoverCostRate`). Checking one family alone and falling back to another would recover a rate from output tokens and then apply it to total throughput; the existing `maxInteractivityAtCost` tests caught exactly that mistake. - -### Impact on published numbers - -Costs come down, since they were the overstated side. Over 555 unclamped reads -(dsr1 8k/1k, targets 20-75), new/old: - -| metric | p05 | p50 | p95 | min | within 1% | within 10% | -| -------------- | ----- | ----- | ----- | ----- | --------- | ---------- | -| $/M tok total | 0.379 | 0.963 | 1.026 | 0.039 | 19.1% | 66.5% | -| $/M tok output | 0.220 | 0.948 | 1.026 | 0.020 | 17.8% | 61.3% | -| $/M tok input | 0.553 | 0.976 | 1.026 | 0.100 | 27.6% | 75.0% | - -Two-thirds of reads move under 10%; the tail is the sparsely swept disaggregated -configs, which is also where the old numbers were least defensible. diff --git a/packages/app/src/components/calculator/interpolation.ts b/packages/app/src/components/calculator/interpolation.ts index d2b3abee9..4a408450a 100644 --- a/packages/app/src/components/calculator/interpolation.ts +++ b/packages/app/src/components/calculator/interpolation.ts @@ -168,17 +168,12 @@ export function getCostField( * the points avoids threading the hardware registry into this module, which must * stay dependency-free for the Python port. * - * Why this exists: splining such a metric directly is wrong. It averages - * reciprocals, and `1/x` is convex, so the interpolated metric diverges from the - * value implied by the interpolated throughput — by `(1+r)^2/(4r)` for a knot - * pair whose throughputs differ by `r`, and `r` reaches ~100 on sparsely swept - * frontiers. Measured on real frontiers, the splined read is the higher one - * 73.6% of the time (median 1.057, max 25.3, min 0.95 — Steffen slopes let the - * cubic undershoot the chord in some brackets, so the direction is usual rather - * than universal). Deriving from the interpolated throughput preserves the - * identity `metric x throughput = c` that the per-point values on /inference obey - * by construction, and lands ~4x closer to held-out measured points. See - * docs/tco-calculator.md. + * Why this exists: independently splining the reciprocal metric and throughput + * produces two curves that need not satisfy `metric x throughput = c` between + * measured knots. Deriving from the interpolated throughput preserves that + * defining identity. The numerical effect depends on frontier density and can + * move either direction with Steffen splines; see docs/tco-calculator.md for a + * dated, reproducible measurement against the live API. * * Returns null unless EVERY usable point agrees on the constant. That check is * the safety rail: the identity is what licenses re-deriving the metric, so a diff --git a/packages/app/src/components/calculator/useThroughputData.test.ts b/packages/app/src/components/calculator/useThroughputData.test.ts index e439ef6e2..eff301846 100644 --- a/packages/app/src/components/calculator/useThroughputData.test.ts +++ b/packages/app/src/components/calculator/useThroughputData.test.ts @@ -1,3 +1,6 @@ +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; + import { describe, expect, it } from 'vitest'; import type { BenchmarkRow } from '@/lib/api'; @@ -17,6 +20,21 @@ import { sign, } from './useThroughputData'; +const PYTHON_INTERPOLATION_HELPER = resolve( + import.meta.dirname, + '../../../../..', + '.claude/skills/write-inferencex-blog/iso_interactivity.py', +); + +function interpolateWithPython(request: Record): number | null { + const result = spawnSync('python3', [PYTHON_INTERPOLATION_HELPER], { + input: JSON.stringify(request), + encoding: 'utf8', + }); + if (result.status !== 0) throw new Error(`Python interpolation failed: ${result.stderr}`); + return (JSON.parse(result.stdout) as { value: number | null }).value; +} + // --------------------------------------------------------------------------- // Test fixtures // --------------------------------------------------------------------------- @@ -1393,11 +1411,40 @@ describe('interpolateForGPU cost derivation', () => { } }); - it('matches the Python port bundled with the blog skill', () => { - // iso_interactivity.py, same frontier and target, reciprocal_of=throughput. - // Drift here means blog tables diverge from the live chart — see AGENTS.md. - const r = interpolateForGPU(frontier, 42, 'interactivity_to_throughput', 'costh')!; - expect(r.cost).toBeCloseTo(0.44517072882391184, 12); + it('matches the Python blog helper', () => { + const target = 42; + const typescriptValue = interpolateForGPU( + frontier, + target, + 'interactivity_to_throughput', + 'costh', + )!.cost; + const pythonValue = interpolateWithPython({ + points: frontier.map((point) => ({ + interactivity: point.interactivity, + throughput: point.throughput, + cost: point.costh, + })), + target_iv: target, + metric_key: 'cost', + reciprocal_of: 'throughput', + }); + + expect(pythonValue).toBeCloseTo(typescriptValue, 14); + }); + + it('makes the Python helper return null when reciprocal throughput is missing', () => { + const pythonValue = interpolateWithPython({ + points: [ + { interactivity: 10, throughput: 1000, output_throughput: 800, joules: 2 }, + { interactivity: 30, throughput: 500, joules: 4 }, + ], + target_iv: 20, + metric_key: 'joules', + reciprocal_of: 'output_throughput', + }); + + expect(pythonValue).toBeNull(); }); it('is exact at a measured point, where both methods agree anyway', () => { diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index 78b0a3439..90e7c5af3 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -98,8 +98,8 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { /** * Metrics defined as a per-chip constant divided by a throughput, mapped to the * throughput metric they divide. These are interpolated by splining that - * throughput and re-deriving, never by splining the metric itself — `1/x` is - * convex, so splining it overstates the value between measured points. See + * throughput and re-deriving, never by splining the metric independently, so + * the interpolated pair preserves `metric x throughput = constant`. See * `recoverReciprocalNumerator` and docs/tco-calculator.md. * * The `measured*` energy keys are deliberately absent: their numerator is