Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -645,7 +645,7 @@ export default function ChartDisplay() {
if (!derivedMetrics) {
// Legacy AgentX axes can still render transient/non-persisted rows, which
// have no ids to request. Persisted rows remain gated on their derived
// metrics so every displayed frontier keeps the canonical iff contract.
// metrics so every displayed frontier can enforce canonical eligibility.
if (!derivedSpec && derivedTargetIds.length === 0) return visibleGraphs;
return visibleGraphs.map((graph) => ({ ...graph, data: [], clippedData: [] }));
}
Expand Down
8 changes: 5 additions & 3 deletions packages/app/src/components/inference/ui/GPUGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import {
paretoFrontForDirection,
type ParetoDirection,
} from '@/lib/chart-utils';
import { canonicalFrontierPoints } from '@/components/inference/utils/canonicalFrontier';
import { canonicalParetoIntersection } from '@/components/inference/utils/canonicalFrontier';
import type {
ChartDefinition,
InferenceData,
Expand Down Expand Up @@ -256,8 +256,10 @@ const GPUGraph = React.memo(
const dir = chartDefinition[rooflineKey] as ParetoDirection | undefined;
const frontier = paretoFrontForDirection(dir ?? 'lower_right');
for (const key of Object.keys(groupedData)) {
const canonicalPoints = canonicalFrontierPoints(groupedData[key]);
result[key] = canonicalPoints ?? frontier(groupedData[key].filter(isFrontierEligible));
const canonicalPoints = canonicalParetoIntersection(groupedData[key], dir ?? 'lower_right');
result[key] = (
canonicalPoints ?? frontier(groupedData[key].filter(isFrontierEligible))
).toSorted((a, b) => a.x - b.x);
}
return result;
}, [groupedData, selectedYAxisMetric, chartDefinition]);
Expand Down
16 changes: 8 additions & 8 deletions packages/app/src/components/inference/ui/ScatterGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ import {
paretoFrontForDirection,
type ParetoDirection,
} from '@/lib/chart-utils';
import { canonicalFrontierPoints } from '@/components/inference/utils/canonicalFrontier';
import { canonicalParetoIntersection } from '@/components/inference/utils/canonicalFrontier';
import { type RooflineDirection, getSpeedOverlayCorners } from '@/lib/speed-overlay';
import type {
ChartDefinition,
Expand Down Expand Up @@ -395,7 +395,7 @@ const SCATTER_STRINGS = {
logScale: 'Log Scale',
optimalOnly: 'Optimal Only',
optimalInfo:
'On agentic, optimal is defined by the E2E Normalized Interactivity Pareto frontier. Every x-axis reuses exactly that winner set.',
'On agentic, optimal points must be Pareto-optimal on the selected x-axis and also belong to the E2E Normalized Interactivity frontier.',
labels: 'Labels',
highContrast: 'High Contrast',
parallelismLabels: 'Parallelism Labels',
Expand All @@ -410,7 +410,7 @@ const SCATTER_STRINGS = {
logScale: '对数缩放',
optimalOnly: '仅最优',
optimalInfo:
'在智能体场景中,最优点由端到端归一化交互性的 Pareto 前沿统一定义,所有横轴均复用同一组优胜点。',
'在智能体场景中,最优点既必须在当前横轴上满足 Pareto 最优,也必须属于端到端归一化交互性的 Pareto 前沿。',
labels: '标签',
highContrast: '高对比度',
parallelismLabels: '并行配置标签',
Expand Down Expand Up @@ -780,10 +780,10 @@ const ScatterGraph = React.memo(
for (const hwKey of Object.keys(groupedData)) {
const combined: InferenceData[] = [];
for (const datePoints of groupPointsByDate(groupedData[hwKey]).values()) {
// Agentic modes reuse the exact normalized north-star winners.
// Do not Pareto them a second time after swapping x axes: doing so
// would violate the iff contract by dropping canonical winners.
const canonicalPoints = canonicalFrontierPoints(datePoints);
// Agentic modes intersect the selected-axis Pareto frontier with the
// normalized north-star frontier. This keeps every drawn curve a true
// Pareto frontier without admitting a non-canonical winner.
const canonicalPoints = canonicalParetoIntersection(datePoints, dir ?? 'lower_right');
const front = canonicalPoints ?? frontierFn(datePoints.filter(isFrontierEligible));
if (front.length === 0) continue;
combined.push(...front);
Expand Down Expand Up @@ -1020,7 +1020,7 @@ const ScatterGraph = React.memo(
const frontierFn = paretoFrontForDirection(dir ?? 'lower_right');
const result: Record<string, Entry> = {};
for (const [key, group] of Object.entries(grouped)) {
const canonicalPoints = canonicalFrontierPoints(group.points);
const canonicalPoints = canonicalParetoIntersection(group.points, dir ?? 'lower_right');
const front = canonicalPoints ?? frontierFn(group.points.filter(isFrontierEligible));
front.sort((a, b) => a.x - b.x);
result[key] = { hwKey: group.hwKey, runIndex: group.runIndex, points: front };
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { describe, expect, it } from 'vitest';

import type { InferenceData } from '../types';
import { canonicalFrontierPoints, canonicalNormalizedFrontierIds } from './canonicalFrontier';
import { canonicalNormalizedFrontierIds, canonicalParetoIntersection } from './canonicalFrontier';

const point = (id: number, y: number, over: Partial<InferenceData> = {}): InferenceData =>
({
Expand Down Expand Up @@ -51,19 +51,69 @@ describe('canonicalNormalizedFrontierIds', () => {
});
});

describe('canonicalFrontierPoints', () => {
it('returns the exact stamped set without a second axis-local Pareto pass', () => {
const canonicalA = point(1, 100, { x: 1, isOnNormalizedInteractivityFrontier: true });
const canonicalB = point(2, 90, { x: 2, isOnNormalizedInteractivityFrontier: true });
const localOnly = point(3, 500, { x: 3, isOnNormalizedInteractivityFrontier: false });
describe('canonicalParetoIntersection', () => {
it.each([
[
'E2E latency',
[
[5, 10.78256, 9091],
[10, 11.35504, 17113],
[15, 10.55072, 22505],
[20, 14.05108, 34255],
[30, 17.62118, 46840],
[40, 25.18999, 54350],
[50, 40.75138, 54443],
],
],
[
'TTFT',
[
[5, 1.13862, 9091],
[10, 0.97378, 17113],
[15, 0.77209, 22505],
[20, 1.11642, 34255],
[30, 1.27482, 46840],
[40, 1.50933, 54350],
[50, 2.80568, 54443],
],
],
])('removes the screenshot zig-zag from the %s frontier', (_axis, rows) => {
const points = rows.map(([conc, x, y]) =>
point(conc, y, {
x,
conc,
isOnNormalizedInteractivityFrontier: true,
}),
);

expect(canonicalFrontierPoints([canonicalA, canonicalB, localOnly])).toEqual([
canonicalA,
canonicalB,
]);
expect(
canonicalParetoIntersection(points, 'upper_right')?.map((candidate) => candidate.conc),
).toEqual([15, 20, 30, 40, 50]);
});

it('computes the selected-axis frontier before intersecting the canonical set', () => {
const nonCanonicalDominator = point(1, 110, {
x: 1,
isOnNormalizedInteractivityFrontier: false,
});
const dominatedCanonicalWinner = point(2, 100, {
x: 2,
isOnNormalizedInteractivityFrontier: true,
});
const canonicalTradeoff = point(3, 120, {
x: 3,
isOnNormalizedInteractivityFrontier: true,
});

expect(
canonicalParetoIntersection(
[dominatedCanonicalWinner, canonicalTradeoff, nonCanonicalDominator],
'upper_right',
),
).toEqual([canonicalTradeoff]);
});

it('returns null when no canonical stamp is present', () => {
expect(canonicalFrontierPoints([point(1, 100)])).toBeNull();
expect(canonicalParetoIntersection([point(1, 100)], 'upper_right')).toBeNull();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@
* @description Canonical agentic Pareto-frontier helpers.
*
* E2E Normalized Interactivity is the agentic "north star" axis. Its true
* Pareto frontier is computed once, then the exact same winning benchmark ids
* are re-plotted on E2E latency, Interactivity, and TTFT. Those alternate axes
* must not add a locally-optimal point or drop a canonical winner with a second
* Pareto pass.
* Pareto frontier supplies the canonical eligibility set for every agentic
* x-axis. Each displayed axis still computes its own true Pareto frontier, and
* only points that belong to both sets may be drawn as optimal.
*/
import type { DerivedAgenticMetricMap } from '@/hooks/api/use-derived-agentic-metrics';
import { isPersistedBenchmarkId } from '@/lib/benchmark-id';
Expand Down Expand Up @@ -69,16 +68,27 @@ export function canonicalNormalizedFrontierIds(
}

/**
* Return the exact canonical winners when frontier flags are present.
* Return the intersection of the selected-axis Pareto frontier and the
* normalized-interactivity frontier when canonical flags are present.
*
* The selected-axis frontier must be computed from all eligible points before
* applying the canonical restriction. Computing Pareto(canonicalPoints) would
* incorrectly retain a canonical point that is dominated on the selected axis
* by a non-canonical point.
*
* `null` means no canonical restriction was stamped, so the caller should
* compute its ordinary local Pareto frontier.
*/
export function canonicalFrontierPoints(points: InferenceData[]): InferenceData[] | null {
export function canonicalParetoIntersection(
points: InferenceData[],
direction: ParetoDirection,
): InferenceData[] | null {
const isCanonical = points.some(
(point) => point.isOnNormalizedInteractivityFrontier !== undefined,
);
if (!isCanonical) return null;
return points.filter(
(point) => point.isOnNormalizedInteractivityFrontier === true && isFrontierEligible(point),
const selectedAxisFrontier = paretoFrontForDirection(direction)(
points.filter(isFrontierEligible),
);
return selectedAxisFrontier.filter((point) => point.isOnNormalizedInteractivityFrontier === true);
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
} from '@/lib/chart-utils';

import type { ClippedInferenceData, InferenceData } from '../types';
import { canonicalFrontierPoints } from './canonicalFrontier';
import { canonicalParetoIntersection } from './canonicalFrontier';

export interface FrontierContinuation {
from: InferenceData;
Expand Down Expand Up @@ -36,7 +36,7 @@ export function buildFrontierContinuations(
const visibleSet = new Set(visible);
const clippedByPoint = new Map(clipped.map((entry) => [entry.point, entry]));
const allPoints = [...visible, ...clipped.map((entry) => entry.point)];
const canonicalPoints = canonicalFrontierPoints(allPoints);
const canonicalPoints = canonicalParetoIntersection(allPoints, direction);
const frontier = (
canonicalPoints ?? paretoFrontForDirection(direction)(allPoints.filter(isFrontierEligible))
).toSorted((a, b) => a.x - b.x);
Expand Down