diff --git a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts new file mode 100644 index 00000000..68803f42 --- /dev/null +++ b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts @@ -0,0 +1,163 @@ +/** + * Overlay (unofficial run) points must respect the "Optimal Only" toggle the + * same way official points do. + * + * Regression: with Optimal Only ON, official non-pareto points are hidden via + * `isPointVisible`, but overlay X markers rendered every point unconditionally. + * On the agentic interactivity chart this made an e2e-dominated config (TP8 + * C=4 in the GLM5.2 B300 hicache run) look like a pareto point: its X marker + * stayed visible sitting on the dashed roofline (the monotone spline between + * C=8 and C=2 passes within ~0.5% of it) while the official twin was hidden. + * + * Fixture values are the real run-29682242847 numbers: + * conc, p90_intvty, tput_per_gpu, p90_e2el + * C=4 is dominated on e2e by C=8 (12874 tok/s @ 33.1s vs 9415 @ 48.0s), so + * with Optimal Only ON exactly 4 of the 5 overlay X's must stay visible. + */ +import { unlockAgenticGate } from '../support/e2e'; + +const DEFAULT_MODEL_DB_KEY = 'dsv4'; +const AGENTIC_DATE = '2026-07-19'; +const OVERLAY_RUN_ID = '29682242847'; +const OVERLAY_RUN_URL = `https://github.com/SemiAnalysisAI/InferenceX/actions/runs/${OVERLAY_RUN_ID}`; + +const REAL_CONFIGS: [number, number, number, number][] = [ + [48, 10.6, 17199, 126.9], + [8, 68.5, 12874, 33.1], + [4, 88.3, 9415, 48], // e2e-dominated by C=8 → NOT optimal + [2, 111.1, 5018, 30], + [1, 130.2, 2600, 25.8], +]; + +const metricsFor = (intvty: number, tput: number, e2el: number): Record => ({ + median_itl: 1 / (intvty * 1.2), + p90_itl: 1 / intvty, + p99_itl: 1 / (intvty * 0.8), + median_e2el: e2el * 0.8, + p90_e2el: e2el, + p99_e2el: e2el * 1.3, + median_ttft: 0.5, + p90_ttft: 1, + p99_ttft: 2, + tput_per_gpu: tput, + output_tput_per_gpu: tput * 0.3, + input_tput_per_gpu: tput * 0.7, +}); + +let idCursor = 900000; +const b300Rows = (runUrl: string | null) => + REAL_CONFIGS.map(([conc, intvty, tput, e2el]) => ({ + id: runUrl ? 0 : idCursor++, + hardware: 'b300', + framework: 'sglang', + model: DEFAULT_MODEL_DB_KEY, + precision: 'fp4', + spec_method: 'none', + disagg: false, + is_multinode: false, + prefill_tp: 8, + decode_tp: 8, + num_prefill_gpu: 8, + num_decode_gpu: 8, + isl: null, + osl: null, + conc, + offload_mode: 'on', + benchmark_type: 'agentic_traces', + image: 'sglang:test', + metrics: metricsFor(intvty, tput, e2el), + workers: null, + date: AGENTIC_DATE, + run_url: runUrl, + })); + +const availability = [ + { + model: DEFAULT_MODEL_DB_KEY, + isl: null, + osl: null, + precision: 'fp4', + hardware: 'b300', + framework: 'sglang', + spec_method: 'none', + disagg: false, + benchmark_type: 'agentic_traces', + date: AGENTIC_DATE, + }, +]; + +const countVisible = ($els: JQuery): number => + [...$els].filter((el) => getComputedStyle(el).opacity !== '0').length; + +describe('Overlay points respect Optimal Only (agentic interactivity)', () => { + before(() => { + cy.intercept('GET', '/api/v1/availability', { body: availability }).as('availability'); + cy.intercept('GET', '/api/v1/benchmarks*', { body: b300Rows(null) }).as('benchmarks'); + cy.intercept('GET', '/api/unofficial-run*', { + body: { + runInfos: [ + { + id: OVERLAY_RUN_ID, + name: 'add-glm5.2-b300-agentic-hicache', + branch: 'add-glm5.2-b300-agentic-hicache', + sha: 'abc000', + createdAt: `${AGENTIC_DATE}T00:00:00Z`, + url: OVERLAY_RUN_URL, + conclusion: 'success', + status: 'completed', + isNonMainBranch: true, + }, + ], + benchmarks: b300Rows(OVERLAY_RUN_URL), + evaluations: [], + }, + }).as('unofficialRun'); + cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { + onBeforeLoad(win) { + win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); + unlockAgenticGate(win); + }, + }); + cy.wait('@unofficialRun'); + cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( + 'have.attr', + 'aria-selected', + 'true', + ); + cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should( + 'have.length', + REAL_CONFIGS.length, + ); + }); + + // Optimal Only defaults ON (i_optimal !== '0') — the DEFAULT view is where + // the regression lived: official C=4 hidden, overlay C=4 X still drawn. + it('hides the e2e-dominated overlay point in the default Optimal Only view', () => { + cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked'); + // Official parity check: 4 of 5 official dots visible. + cy.get('[data-testid="inference-chart-display"] svg .dot-group').then(($dots) => { + expect(countVisible($dots), 'visible official points').to.eq(REAL_CONFIGS.length - 1); + }); + // The overlay must hide its C=4 too — 4 of 5 X markers visible. + cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').then(($pts) => { + expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length - 1); + }); + }); + + it('shows all overlay points when Optimal Only is turned off', () => { + cy.get('#scatter-hide-non-optimal').click(); + cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'unchecked'); + cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').then(($pts) => { + expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length); + }); + }); + + it('re-hides the e2e-dominated overlay point when Optimal Only is re-enabled', () => { + cy.get('#scatter-hide-non-optimal').click(); + cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked'); + cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').then(($pts) => { + expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length - 1); + }); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index 3ae6c323..197fcd6b 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -23,15 +23,12 @@ import { getModelSortIndex, hardwareKeyMatchesAnyBase, } from '@/lib/constants'; -import { - mergeRunScopedRows, - transformBenchmarkRows, - withPercentile, -} from '@/lib/benchmark-transform'; +import { mergeRunScopedRows, transformBenchmarkRows } from '@/lib/benchmark-transform'; import { Sequence, type Model } from '@/lib/data-mappings'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { calculateCostsForGpus, calculatePowerForGpus } from '@/lib/utils'; -import { paretoFrontForDirection, type ParetoDirection } from '@/lib/chart-utils'; +import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; +import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; import { applyQuickFilters, computeAvailableQuickFilters, @@ -86,39 +83,14 @@ function e2eParetoIds( selectedYAxisMetric: string, percentile: string, ): Set | null { - const e2eChartDef = (chartDefinitions as ChartDefinition[]).find((c) => c.chartType === 'e2e'); - if (!e2eChartDef) return null; - const dir = e2eChartDef[`${selectedYAxisMetric}_roofline` as keyof ChartDefinition] as - | ParetoDirection - | undefined; - if (!dir) return null; - const frontierFn = paretoFrontForDirection(dir); - // Percentile-prefixed e2e-latency field name (e.g. 'p90_e2el'). - const e2elField = withPercentile('median_e2el', percentile); - const metricKey = selectedYAxisMetric.replace('y_', '') as YAxisMetricKey; - - // Re-frame each candidate point in (e2el, y) space, then compute the - // pareto per (hwKey, precision, date) bucket — frontiers don't span dates - // (a May 17 point can't dominate a May 15 plot). - const byGroup = new Map(); - for (const p of points) { - const yValue = (p[metricKey] as { y?: number } | undefined)?.y; - const xValue = (p as unknown as Record)[e2elField]; - if (typeof xValue !== 'number' || !Number.isFinite(xValue)) continue; - if (typeof yValue !== 'number' || !Number.isFinite(yValue)) continue; - const key = `${p.hwKey}|${p.precision}|${p.date}`; - let bucket = byGroup.get(key); - if (!bucket) { - bucket = []; - byGroup.set(key, bucket); - } - bucket.push({ ...p, x: xValue, y: yValue }); - } + // Shared seed with the overlay path (processOverlayChartData) so both draw + // the SAME e2e-restricted frontier. null = the y-metric has no e2e roofline + // direction → caller skips filtering. Only persisted DB rows carry ids to pin. + const winners = e2eFrontierWinners(points, selectedYAxisMetric, percentile); + if (winners === null) return null; const ids = new Set(); - for (const bucket of byGroup.values()) { - for (const f of frontierFn(bucket)) { - if (isPersistedBenchmarkId(f.id)) ids.add(f.id); - } + for (const winner of winners) { + if (isPersistedBenchmarkId(winner.id)) ids.add(winner.id); } return ids; } @@ -399,81 +371,49 @@ export function useChartData( (chartDefinitions as ChartDefinition[]).map((chartDef) => { const metricKey = selectedYAxisMetric.replace('y_', '') as YAxisMetricKey; - // Default x-axis = chart's natural latency metric, percentile-adjusted - // for the agentic case (median_e2el → p99_e2el etc.). Percentiles only - // exist for agentic scenarios; fixed-seq benchmarks have no p90_/p99_ - // columns, and the percentile selector is hidden for them — but its - // state persists at the 'p90' default across mode/sequence switches. - // Applying that stale percentile to a fixed-seq metric yields a null - // column (e.g. p90_intvty), which renders as x=0/NaN and drops the - // point from the chart. Force median for non-agentic so the natural - // metric (median_intvty / median_e2el) is used. + // Resolve which data field the x-axis plots — shared with the overlay + // path (processOverlayChartData) via resolveXAxisField so the two + // can't drift. Labels/headings stay here (display-only) and follow the + // resolver's branch discriminant. const isAgentic = selectedSequence === Sequence.AgenticTraces; - const naturalX = withPercentile( - chartDef.x, - isAgentic ? selectedPercentile : 'median', - ) as keyof AggDataEntry; - let xAxisField: keyof AggDataEntry = naturalX; - let xAxisLabel = chartDef.x_label; - - const metricTitle = - (chartDef[`${selectedYAxisMetric}_title` as keyof ChartDefinition] as string) || ''; - const isInputMetric = metricTitle.toLowerCase().includes('input'); - - // Resolve the effective x-axis override per chart type const effectiveXMetric = chartDef.chartType === 'e2e' ? selectedE2eXAxisMetric : selectedXAxisMetric; - // The TTFT override is now any *_ttft metric (not just p90_ttft) — the - // x-axis-mode picker reconciles the percentile prefix based on sequence - // kind (fixed-seq → median, agentic → user-picked percentile). - const isTtftOverride = - typeof effectiveXMetric === 'string' && effectiveXMetric.endsWith('_ttft'); + const resolved = resolveXAxisField(chartDef, selectedYAxisMetric, effectiveXMetric, { + isAgentic, + percentile: selectedPercentile, + }); + const naturalX = resolved.naturalX as keyof AggDataEntry; + const xAxisField = resolved.xAxisField as keyof AggDataEntry; + const { isTtftOverride } = resolved; + const ttftPctl = isTtftOverride ? (effectiveXMetric as string).replace(/_ttft$/u, '') : 'p90'; const ttftPctlWord = ttftPctl === 'median' ? 'Median' : ttftPctl.toUpperCase(); const ttftLabel = `${ttftPctlWord} Time To First Token (s)`; - if ( - effectiveXMetric && - chartDef.chartType === 'interactivity' && - isInputMetric && - !isAgentic - ) { - xAxisField = effectiveXMetric as keyof AggDataEntry; + let xAxisLabel = chartDef.x_label; + if (resolved.branch === 'user-input-override') { const labelKey = `${selectedYAxisMetric}_x_label` as keyof ChartDefinition; if (effectiveXMetric === chartDef[`${selectedYAxisMetric}_x` as keyof ChartDefinition]) { xAxisLabel = (chartDef[labelKey] as string) || chartDef.x_label; } else { xAxisLabel = isTtftOverride ? ttftLabel : chartDef.x_label; } - } else if (chartDef.chartType === 'interactivity' && isInputMetric) { - // Agentic falls through here too — the manual X-axis dropdown is - // hidden in agentic mode (would double up with the percentile - // selector), so the config default + percentile post-processing - // below drives the x axis. - const xOverrideKey = `${selectedYAxisMetric}_x` as keyof ChartDefinition; + } else if (resolved.branch === 'config-input-override') { const xLabelOverrideKey = `${selectedYAxisMetric}_x_label` as keyof ChartDefinition; - xAxisField = (chartDef[xOverrideKey] as keyof AggDataEntry) || chartDef.x; xAxisLabel = (chartDef[xLabelOverrideKey] as string) || chartDef.x_label; - } else if (chartDef.chartType === 'e2e' && isTtftOverride) { - xAxisField = effectiveXMetric as keyof AggDataEntry; + } else if (resolved.branch === 'e2e-ttft-override') { xAxisLabel = ttftLabel; } - // Agentic: rewrite the resolved x metric to the chosen percentile, - // and relabel accordingly. Both have to be updated unconditionally — - // xAxisField may already be percentile-adjusted (via naturalX) while - // xAxisLabel still carries the raw chartDef.x_label prefix. - // The chart heading ("vs. ") is also rewritten to include - // the percentile so the title above the plot reflects what's drawn. + // Agentic: relabel to the chosen percentile (the resolver already + // rewrote the field) — xAxisLabel still carries the raw chartDef + // prefix. The chart heading ("vs. ") is also rewritten so the + // title above the plot reflects what's drawn. const headingKey = `${selectedYAxisMetric}_heading` as keyof ChartDefinition; let chartHeading = (chartDef[headingKey] as string) || chartDef.heading; if (isAgentic) { - xAxisField = withPercentile( - xAxisField as string, - selectedPercentile, - ) as keyof AggDataEntry; const pctlWord = selectedPercentile.toUpperCase(); xAxisLabel = xAxisLabel.replace(/^(?:Median|Mean|P75|P90|P95|P99(?:\.9)?)\b/iu, pctlWord); chartHeading = chartHeading.replace( diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index cec2072e..76d3e6f4 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -380,12 +380,19 @@ export default function ChartDisplay() { if (!rawData || rawData.data.length === 0) return null; const effectiveXMetric = chartType === 'e2e' ? selectedE2eXAxisMetric : selectedXAxisMetric; + const isAgentic = sequenceKind(selectedSequence) === 'agentic'; const processed = processOverlayChartData( rawData.data, chartType, selectedYAxisMetric, effectiveXMetric, - { isAgentic: sequenceKind(selectedSequence) === 'agentic' }, + { + isAgentic, + selectedPercentile, + // Same gate useChartData applies to the official points — on any + // non-e2e x-mode, agentic rooflines are restricted to e2e winners. + restrictToE2eFrontier: isAgentic && selectedXAxisMode !== 'e2e', + }, ); let overlayPoints = processed; @@ -425,6 +432,8 @@ export default function ChartDisplay() { selectedYAxisMetric, selectedXAxisMetric, selectedE2eXAxisMetric, + selectedPercentile, + selectedXAxisMode, compareGpuPair, ]); diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 80da237d..551a1b2a 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -52,11 +52,8 @@ import { getShapeKeyForPrecision, } from '@/lib/chart-rendering'; import { useThemeColors } from '@/hooks/useThemeColors'; -import { - isFrontierEligible, - paretoFrontForDirection, - type ParetoDirection, -} from '@/lib/chart-utils'; +import { paretoFrontForDirection, type ParetoDirection } from '@/lib/chart-utils'; +import { e2eRestrictedSeed } from '@/components/inference/utils/e2eFrontier'; import { type RooflineDirection, getSpeedOverlayCorners } from '@/lib/speed-overlay'; import type { ChartDefinition, @@ -659,17 +656,9 @@ const ScatterGraph = React.memo( for (const hwKey of Object.keys(groupedData)) { const combined: InferenceData[] = []; for (const datePoints of groupPointsByDate(groupedData[hwKey]).values()) { - // In non-e2e xmodes, useChartData stamps every point with an - // `isOnE2eFrontier` flag so the line is restricted to the - // e2e-Pareto winners — same set of points across every chart, - // just re-plotted at the chosen x metric. When the flag is - // present on ANY point in the bucket, narrow to the winners - // before paretoing (otherwise we'd recompute a fresh frontier - // on the swapped x axis and reintroduce the benchmark hack). - const flagged = datePoints.some((p) => p.isOnE2eFrontier !== undefined); - const seedPoints = ( - flagged ? datePoints.filter((p) => p.isOnE2eFrontier === true) : datePoints - ).filter(isFrontierEligible); + // e2eRestrictedSeed narrows to the e2e-Pareto winners when the + // isOnE2eFrontier flag is present (agentic non-e2e xmodes). + const seedPoints = e2eRestrictedSeed(datePoints); if (seedPoints.length === 0) continue; combined.push(...frontierFn(seedPoints)); } @@ -791,13 +780,40 @@ const ScatterGraph = React.memo( const frontierFn = paretoFrontForDirection(dir ?? 'lower_right'); const result: Record = {}; for (const [key, group] of Object.entries(grouped)) { - const front = frontierFn(group.points.filter(isFrontierEligible)); + // Same e2e-winner narrowing the official `rooflines` memo applies + // (flags stamped per run in processOverlayChartData). + const front = frontierFn(e2eRestrictedSeed(group.points)); front.sort((a, b) => a.x - b.x); result[key] = { hwKey: group.hwKey, runIndex: group.runIndex, points: front }; } return result; }, [processedOverlayData, selectedYAxisMetric, chartDefinition, runIndexByUrl]); + // Overlay counterpart of `optimalPointKeys`: the points on any overlay + // run's drawn roofline (already e2e-restricted for agentic non-e2e modes). + // Frontier arrays hold the same object references as `processedOverlayData` + // items — the pareto fns return the refs they're handed — so identity + // membership is exact, and unlike composite string keys it can't collide + // across runs sharing a (hw, precision, tp, conc) tuple. + const overlayOptimalPoints = useMemo(() => { + const set = new Set(); + for (const group of Object.values(overlayRooflines)) { + for (const p of group.points) set.add(p); + } + return set; + }, [overlayRooflines]); + + // Overlay points respect the Optimal Only toggle exactly like official + // points do — "optimal" = on the overlay run's drawn roofline. Without + // this, an e2e-dominated overlay config (hidden on the official side) kept + // its X marker sitting on the dashed roofline and read as a pareto point. + // Hardware/precision/quick filters are applied upstream in + // `processedOverlayData`, so optimality is the only condition here. + const isOverlayPointVisible = useCallback( + (d: InferenceData) => !hideNonOptimal || overlayOptimalPoints.has(d), + [hideNonOptimal, overlayOptimalPoints], + ); + // All official points for rendering (unfiltered — visibility via opacity) const pointsData = useMemo(() => Object.values(groupedData).flat(), [groupedData]); @@ -840,11 +856,13 @@ const ScatterGraph = React.memo( }; } const { runIndex, runId, branch } = pointsTableTarget; - // Overlay series: this run's points, respecting the overlay hw toggles. + // Overlay series: this run's points, respecting the overlay hw toggles + // and Optimal Only (same visibility filters as the official branch above). const pts = processedOverlayData.filter( (p) => overlayRunIndex(p.run_url ?? null, runIndexByUrl) === runIndex && - activeOverlayHwTypes.has(p.hwKey as string), + activeOverlayHwTypes.has(p.hwKey as string) && + isOverlayPointVisible(p), ); return { hw: `overlay-run-${runId}`, @@ -860,6 +878,7 @@ const ScatterGraph = React.memo( selectedPrecisions, hideNonOptimal, optimalPointKeys, + isOverlayPointVisible, resolveColor, processedOverlayData, runIndexByUrl, @@ -907,8 +926,17 @@ const ScatterGraph = React.memo( if (hideNonOptimal) { pts = pts.filter((d) => optimalPointKeys.has(optimalPointKey(d))); } - return processedOverlayData.length > 0 ? [...pts, ...processedOverlayData] : pts; - }, [filteredData, processedOverlayData, hideNonOptimal, optimalPointKeys]); + // Overlay points hidden by Optimal Only are excluded from the domain too + // so hidden outliers don't stretch the axes. + const overlayPts = processedOverlayData.filter(isOverlayPointVisible); + return overlayPts.length > 0 ? [...pts, ...overlayPts] : pts; + }, [ + filteredData, + processedOverlayData, + hideNonOptimal, + optimalPointKeys, + isOverlayPointVisible, + ]); const isInputTputMetric = selectedYAxisMetric === 'y_inputTputPerGpu'; @@ -1021,6 +1049,7 @@ const ScatterGraph = React.memo( // Handlers". const interactionRef = useRef({ isPointVisible, + isOverlayPointVisible, effectiveActiveHwTypes, selectedPrecisions, activeOverlayHwTypes, @@ -1030,6 +1059,7 @@ const ScatterGraph = React.memo( }); interactionRef.current = { isPointVisible, + isOverlayPointVisible, effectiveActiveHwTypes, selectedPrecisions, activeOverlayHwTypes, @@ -2118,6 +2148,15 @@ const ScatterGraph = React.memo( overlayPoints.attr('transform', (d) => `translate(${xScale(d.x)},${yScale(d.y)})`); overlayPoints.style('filter', null); + // Optimal Only parity with official points (see isOverlayPointVisible). + // Read through the interaction ref so this long-lived closure sees + // the current toggle state on zoom/label re-renders. + overlayPoints.each(function (d) { + const visible = interactionRef.current.isOverlayPointVisible(d); + d3.select(this) + .style('opacity', visible ? 1 : 0) + .style('pointer-events', visible ? 'auto' : 'none'); + }); overlayPoints .select('.overlay-x') .attr('stroke', (d) => @@ -2591,6 +2630,16 @@ const ScatterGraph = React.memo( sel.select('.tracked-ring').attr('stroke', color); }); + // Overlay X markers: Optimal Only visibility (mirrors the official dot + // loop above — the overlay layer render applies the same predicate, but + // a toggle flip must restyle existing DOM without a chart rebuild). + zoomGroup.selectAll('.unofficial-overlay-pt').each(function (d) { + const visible = ir.isOverlayPointVisible(d); + d3.select(this) + .style('opacity', visible ? 1 : 0) + .style('pointer-events', visible ? 'auto' : 'none'); + }); + // Rooflines: visibility + solid-stroke recolor as direct writes (never // `d`). Gradient strokes keep their url(#…) reference — gradient stop // colors come from the fixed parallelism palette and don't change with @@ -2656,6 +2705,7 @@ const ScatterGraph = React.memo( } }, [ isPointVisible, + isOverlayPointVisible, effectiveActiveHwTypes, selectedPrecisions, activeOverlayHwTypes, diff --git a/packages/app/src/components/inference/utils.test.ts b/packages/app/src/components/inference/utils.test.ts index 7d5b1482..35e48ad2 100644 --- a/packages/app/src/components/inference/utils.test.ts +++ b/packages/app/src/components/inference/utils.test.ts @@ -279,4 +279,201 @@ describe('processOverlayChartData', () => { expect(result).toHaveLength(1); expect(result[0].y).toBe(0.5); }); + + // Regression: overlay points must sit on the SAME x column as the official run. + // useChartData plots agentic interactivity at withPercentile('median_intvty', + // selectedPercentile) (e.g. p90_intvty). The overlay previously ignored the + // percentile and used the raw median_intvty, so an `?unofficialrun=` overlay of + // the very same run rendered to the right of its own official points on the + // "P90 Interactivity" chart. See InferenceX_GLM.png misalignment report. + it('applies the selected percentile to the natural interactivity x-axis for agentic overlays', () => { + const data = [ + pt({ tpPerGpu: { y: 42, roof: false }, median_intvty: 200, p90_intvty: 130 } as any), + ]; + const result = processOverlayChartData(data, 'interactivity', 'y_tpPerGpu', null, { + isAgentic: true, + selectedPercentile: 'p90', + }); + expect(result).toHaveLength(1); + // Must land on p90_intvty (130), NOT the raw median_intvty (200). + expect(result[0].x).toBe(130); + }); + + it('applies the selected percentile to the natural e2e x-axis for agentic overlays', () => { + const data = [pt({ tpPerGpu: { y: 42, roof: false }, median_e2el: 2.5, p99_e2el: 9 } as any)]; + const result = processOverlayChartData(data, 'e2e', 'y_tpPerGpu', null, { + isAgentic: true, + selectedPercentile: 'p99', + }); + expect(result).toHaveLength(1); + expect(result[0].x).toBe(9); + }); + + it('keeps the natural median x-axis for non-agentic overlays regardless of percentile', () => { + // Fixed-seq rows have no p90_/p99_ columns; the percentile selector is hidden + // and forced to median. A stale 'p90' must NOT be applied to fixed-seq overlays. + const data = [ + pt({ tpPerGpu: { y: 42, roof: false }, median_intvty: 200, p90_intvty: 130 } as any), + ]; + const result = processOverlayChartData(data, 'interactivity', 'y_tpPerGpu', null, { + isAgentic: false, + selectedPercentile: 'p90', + }); + expect(result).toHaveLength(1); + expect(result[0].x).toBe(200); + }); + + // Anti-benchmark-hacking parity: the agentic interactivity roofline is + // restricted to configs that ALSO win on end-to-end latency. The overlay must + // stamp `isOnE2eFrontier` the same way the official path does, so + // overlayRooflines draws the same e2e-restricted frontier instead of a fresh + // interactivity-plane one that rides above the official line. See uno.png. + it('stamps isOnE2eFrontier on agentic interactivity overlays (restricts to e2e-Pareto winners)', () => { + // e2e roofline for tpPerGpu is upper_right on (e2el, tput). With e2el 1/2/3 + // and tput 100/200/150, the frontier keeps A(1,100) and B(2,200); C(3,150) + // is dominated (lower tput at higher latency) → NOT on the e2e frontier, + // even though its higher interactivity would put it on a naive intvty front. + const A = pt({ + tpPerGpu: { y: 100, roof: false }, + p90_e2el: 1, + p90_intvty: 130, + } as any); + const B = pt({ + tpPerGpu: { y: 200, roof: false }, + p90_e2el: 2, + p90_intvty: 90, + } as any); + const C = pt({ + tpPerGpu: { y: 150, roof: false }, + p90_e2el: 3, + p90_intvty: 200, + } as any); + const result = processOverlayChartData([A, B, C], 'interactivity', 'y_tpPerGpu', null, { + isAgentic: true, + selectedPercentile: 'p90', + restrictToE2eFrontier: true, + }); + const frontierByY = Object.fromEntries(result.map((p) => [p.y, p.isOnE2eFrontier])); + expect(frontierByY[100]).toBe(true); // A + expect(frontierByY[200]).toBe(true); // B + expect(frontierByY[150]).toBe(false); // C — interactivity-optimal but not e2e-optimal + }); + + it('does not stamp isOnE2eFrontier for non-agentic overlays', () => { + // ChartDisplay computes restrictToE2eFrontier = isAgentic && mode !== 'e2e', + // so fixed-seq always passes false. + const data = [pt({ tpPerGpu: { y: 100, roof: false }, median_intvty: 50, p90_e2el: 1 } as any)]; + const result = processOverlayChartData(data, 'interactivity', 'y_tpPerGpu', null, { + isAgentic: false, + selectedPercentile: 'median', + restrictToE2eFrontier: false, + }); + expect(result[0].isOnE2eFrontier).toBeUndefined(); + }); + + it('does not stamp isOnE2eFrontier in the e2e x-mode (it already IS the e2e frontier)', () => { + const data = [pt({ tpPerGpu: { y: 100, roof: false }, median_e2el: 1, p90_e2el: 1 } as any)]; + const result = processOverlayChartData(data, 'e2e', 'y_tpPerGpu', null, { + isAgentic: true, + selectedPercentile: 'p90', + restrictToE2eFrontier: false, + }); + expect(result[0].isOnE2eFrontier).toBeUndefined(); + }); + + it('stamps isOnE2eFrontier on the e2e chart when x is overridden to TTFT (ttft mode)', () => { + // The 'ttft' x-axis mode renders the e2e chartType with a *_ttft override. + // Official stamps the e2e-frontier flag for every non-e2e x-mode, so the + // overlay must too — otherwise the TTFT overlay roofline draws a fresh + // TTFT-plane frontier instead of the e2e-restricted one. + const A = pt({ + tpPerGpu: { y: 100, roof: false }, + p90_e2el: 1, + p90_ttft: 0.2, + } as any); + const B = pt({ + tpPerGpu: { y: 150, roof: false }, + p90_e2el: 3, // dominated on e2e? No — higher tput at higher e2el stays on upper_right + p90_ttft: 0.4, + } as any); + const C = pt({ + tpPerGpu: { y: 90, roof: false }, + p90_e2el: 5, // dominated: lower tput than B at higher e2el + p90_ttft: 0.1, + } as any); + const result = processOverlayChartData([A, B, C], 'e2e', 'y_tpPerGpu', 'p90_ttft', { + isAgentic: true, + selectedPercentile: 'p90', + restrictToE2eFrontier: true, + }); + const byY = Object.fromEntries(result.map((p) => [p.y, p.isOnE2eFrontier])); + expect(byY[100]).toBe(true); // A + expect(byY[150]).toBe(true); // B + expect(byY[90]).toBe(false); // C — TTFT-optimal but not e2e-optimal + }); + + it('seeds the agentic e2e frontier per unofficial run (runs do not cross-dominate)', () => { + // Merged across runs, run-2's point (higher e2el, lower tput) would be + // dominated by run-1's and dropped. Per run, each is on its own frontier. + const r1 = pt({ + tpPerGpu: { y: 500, roof: false }, + p90_e2el: 1, + p90_intvty: 100, + run_url: 'https://gh/runs/1', + } as any); + const r2 = pt({ + tpPerGpu: { y: 100, roof: false }, + p90_e2el: 5, + p90_intvty: 40, + run_url: 'https://gh/runs/2', + } as any); + const result = processOverlayChartData([r1, r2], 'interactivity', 'y_tpPerGpu', null, { + isAgentic: true, + selectedPercentile: 'p90', + restrictToE2eFrontier: true, + }); + const byUrl = Object.fromEntries(result.map((p) => [p.run_url, p.isOnE2eFrontier])); + expect(byUrl['https://gh/runs/1']).toBe(true); + expect(byUrl['https://gh/runs/2']).toBe(true); // false if runs were merged + }); + + it('leaves isOnE2eFrontier unset for metrics with no e2e roofline direction', () => { + // y_measuredAvgPower has no `_roofline` on the e2e chart def, so no e2e + // restriction applies. The official path leaves the flag undefined and + // draws the roofline unrestricted; the overlay must do the same — an + // all-false stamping here would seed an EMPTY overlay frontier and (with + // Optimal Only on) hide every overlay point. + const data = [ + pt({ + measuredAvgPower: { y: 700, roof: false }, + p90_intvty: 100, + p90_e2el: 10, + } as any), + ]; + const result = processOverlayChartData(data, 'interactivity', 'y_measuredAvgPower', null, { + isAgentic: true, + selectedPercentile: 'p90', + restrictToE2eFrontier: true, + }); + expect(result).toHaveLength(1); + expect(result[0].isOnE2eFrontier).toBeUndefined(); + }); + + it('applies the selected percentile to an agentic input-metric x override', () => { + // Input metrics on the interactivity chart override x to *_ttft; agentic must + // carry the chosen percentile onto that override (p90_ttft) too. + const data = [ + pt({ + inputTputPerGpu: { y: 5, roof: false }, + median_ttft: 0.1, + p90_ttft: 0.4, + } as any), + ]; + const result = processOverlayChartData(data, 'interactivity', 'y_inputTputPerGpu', null, { + isAgentic: true, + selectedPercentile: 'p90', + }); + expect(result).toHaveLength(1); + expect(result[0].x).toBe(0.4); + }); }); diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index f6ebd0f8..2b8074d9 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -5,6 +5,8 @@ */ import chartDefinitions from '@/components/inference/inference-chart-config.json'; +import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; +import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; @@ -81,42 +83,44 @@ export const filterDataByCostLimit = ( /** * Process overlay (unofficial run) data to match the same pipeline as official data. * - * Applies: metric field filtering, x/y remapping (including x-axis overrides for - * input metrics on the interactivity chart), and cost limit filtering. + * Applies: metric field filtering, x/y remapping (via the resolveXAxisField + * resolver shared with `useChartData`, so the overlay of a run lands on the + * identical x column as that run's official points), the e2e-Pareto frontier + * stamping for agentic non-e2e x-modes, and cost limit filtering. + * + * `options.restrictToE2eFrontier` is the caller-computed official gate + * (`isAgentic && selectedXAxisMode !== 'e2e'`) — passed in rather than + * re-derived from chartType so the overlay can't drift from `useChartData`'s + * stamping condition. */ export function processOverlayChartData( data: InferenceData[], chartType: 'e2e' | 'interactivity', selectedYAxisMetric: string, selectedXAxisMetric: string | null, - options?: { isAgentic?: boolean }, + options?: { + isAgentic?: boolean; + selectedPercentile?: string; + restrictToE2eFrontier?: boolean; + }, ): InferenceData[] { const chartDef = (chartDefinitions as ChartDefinition[]).find((d) => d.chartType === chartType); if (!chartDef) return []; const metricKey = selectedYAxisMetric.replace('y_', '') as YAxisMetricKey; const isAgentic = options?.isAgentic === true; + const selectedPercentile = options?.selectedPercentile ?? 'median'; - // Resolve x-axis field (must match useChartData logic) - const metricTitle = - (chartDef[`${selectedYAxisMetric}_title` as keyof ChartDefinition] as string) || ''; - const isInputMetric = metricTitle.toLowerCase().includes('input'); - let xAxisField: string = chartDef.x; // selectedXAxisMetric is already the effective metric for this chart type // (interactivity uses selectedXAxisMetric, e2e uses selectedE2eXAxisMetric). - // Match any *_ttft metric — the x-axis-mode picker can now select any - // percentile (median/p75/p90/p99) depending on sequence kind. - const isTtftOverride = - typeof selectedXAxisMetric === 'string' && selectedXAxisMetric.endsWith('_ttft'); - - if (selectedXAxisMetric && chartDef.chartType === 'interactivity' && isInputMetric) { - xAxisField = selectedXAxisMetric; - } else if (chartDef.chartType === 'interactivity' && isInputMetric) { - const xOverrideKey = `${selectedYAxisMetric}_x` as keyof ChartDefinition; - xAxisField = (chartDef[xOverrideKey] as string) || chartDef.x; - } else if (chartDef.chartType === 'e2e' && isTtftOverride) { - xAxisField = selectedXAxisMetric!; - } + const { xAxisField } = resolveXAxisField(chartDef, selectedYAxisMetric, selectedXAxisMetric, { + isAgentic, + percentile: selectedPercentile, + }); + + // The latency limit targets overload outliers on the TTFT axis only; skip it + // for the natural axis and for agentic (long TTFTs are normal there). + const isTtftX = xAxisField.endsWith('_ttft'); const processedData = data .filter((d) => metricKey in d) @@ -126,14 +130,38 @@ export function processOverlayChartData( return { ...d, x: xValue, y: yValue }; }) .filter( - (d) => - // Skip the latency limit for the natural x-axis or for agentic - // (long TTFTs are normal there, not overload outliers). - xAxisField === chartDef.x || - isAgentic || - !chartDef.y_latency_limit || - d.x <= chartDef.y_latency_limit, + (d) => !isTtftX || isAgentic || !chartDef.y_latency_limit || d.x <= chartDef.y_latency_limit, ); - return filterDataByCostLimit(processedData, chartDef, selectedYAxisMetric); + const costFiltered = filterDataByCostLimit(processedData, chartDef, selectedYAxisMetric); + + // Anti-benchmark-hacking parity: on agentic charts whose x-axis is NOT the + // natural e2e latency, the official roofline is restricted to configs that + // ALSO win on end-to-end latency (useChartData stamps `isOnE2eFrontier`, + // ScatterGraph's rooflines honor it via e2eRestrictedSeed). Stamp the same + // flag on overlay points, seeded per run (matching overlayRooflines' per-run + // grouping) so points from one unofficial run can't dominate another's. + // A null winner set means the y-metric declares no e2e roofline direction — + // no restriction applies, so the flag stays unset (matching the official + // path, which draws those rooflines unrestricted). + if (options?.restrictToE2eFrontier) { + const byRun = new Map(); + for (const p of costFiltered) { + const runKey = p.run_url ?? ''; + let bucket = byRun.get(runKey); + if (!bucket) { + bucket = []; + byRun.set(runKey, bucket); + } + bucket.push(p); + } + for (const runPoints of byRun.values()) { + const winners = e2eFrontierWinners(runPoints, selectedYAxisMetric, selectedPercentile); + // Direction-less metrics resolve null for every run — stop entirely. + if (winners === null) break; + for (const p of runPoints) p.isOnE2eFrontier = winners.has(p); + } + } + + return costFiltered; } diff --git a/packages/app/src/components/inference/utils/e2eFrontier.ts b/packages/app/src/components/inference/utils/e2eFrontier.ts new file mode 100644 index 00000000..588afdd3 --- /dev/null +++ b/packages/app/src/components/inference/utils/e2eFrontier.ts @@ -0,0 +1,113 @@ +/** + * @file e2eFrontier.ts + * @description Shared seed for the anti-benchmark-hacking roofline restriction. + * + * On the non-e2e xmode charts (interactivity, ttft, session-time, prefill-tps) + * the roofline is restricted to the configs that ALSO win on end-to-end latency, + * so a config can't top interactivity while tanking decode (or vice versa). Both + * the official path (`useChartData` → benchmark ids → `isOnE2eFrontier`) and the + * `?unofficialrun=` overlay path (`processOverlayChartData` → `isOnE2eFrontier`) + * MUST seed that restriction identically — otherwise an overlay of the same run + * draws a fresh interactivity-plane frontier that rides above the official + * e2e-restricted line. This helper is that single seed, and + * `e2eRestrictedSeed` is the single interpreter of the resulting flag. + */ +import chartDefinitions from '@/components/inference/inference-chart-config.json'; +import { withPercentile } from '@/lib/benchmark-transform'; +import { + isFrontierEligible, + paretoFrontForDirection, + type ParetoDirection, +} from '@/lib/chart-utils'; + +import type { ChartDefinition, InferenceData } from '../types'; + +/** + * Minimal (e2el, y) projection handed to the pareto functions — they only read + * `.x`/`.y` and return the refs they were given, so a full `{...p}` copy of the + * ~70-property point would be pure allocation waste in two hot memo paths. + * `orig` maps a frontier member back to the caller's point. + */ +interface FramedPoint { + x: number; + y: number; + orig: InferenceData; +} + +/** + * Returns the subset of `points` (by reference) that sit on the (e2e_latency, y) + * Pareto frontier within each (hwKey, precision, date) group — or `null` when + * the e2e chart declares no roofline direction for the selected y-metric (e.g. + * the measured-power metrics), meaning NO restriction applies. Callers must + * treat `null` as "leave `isOnE2eFrontier` unset / draw unrestricted"; an empty + * set means "a restriction applies and nothing qualifies". + * + * The frontier is computed in (e2el, y) space using the e2e chart's roofline + * direction for the selected y-metric and the percentile-prefixed e2e-latency + * field (e.g. `p90_e2el`). + */ +export function e2eFrontierWinners( + points: InferenceData[], + selectedYAxisMetric: string, + percentile: string, +): Set | null { + const e2eChartDef = (chartDefinitions as ChartDefinition[]).find((c) => c.chartType === 'e2e'); + if (!e2eChartDef) return null; + const dir = e2eChartDef[`${selectedYAxisMetric}_roofline` as keyof ChartDefinition] as + | ParetoDirection + | undefined; + if (!dir) return null; + const frontierFn = paretoFrontForDirection(dir); + // Percentile-prefixed e2e-latency field name (e.g. 'p90_e2el'). + const e2elField = withPercentile('median_e2el', percentile); + const metricKey = selectedYAxisMetric.replace('y_', '') as keyof InferenceData; + + // Re-frame each candidate point in (e2el, y) space, then compute the pareto + // per (hwKey, precision, date) bucket — frontiers don't span dates (a May 17 + // point can't dominate a May 15 plot). + const byGroup = new Map(); + for (const p of points) { + const yValue = (p[metricKey] as { y?: number } | undefined)?.y; + const xValue = (p as unknown as Record)[e2elField]; + if (typeof xValue !== 'number' || !Number.isFinite(xValue)) continue; + if (typeof yValue !== 'number' || !Number.isFinite(yValue)) continue; + const key = `${p.hwKey}|${p.precision}|${p.date}`; + let bucket = byGroup.get(key); + if (!bucket) { + bucket = []; + byGroup.set(key, bucket); + } + bucket.push({ x: xValue, y: yValue, orig: p }); + } + const winners = new Set(); + for (const bucket of byGroup.values()) { + // The pareto fns only touch x/y and return input refs — safe on the + // projection. + for (const f of frontierFn(bucket as unknown as InferenceData[])) { + winners.add((f as unknown as FramedPoint).orig); + } + } + return winners; +} + +/** + * Narrow a roofline's seed points to the e2e-Pareto winners when the + * `isOnE2eFrontier` flag is present. + * + * In non-e2e xmodes, agentic points are stamped with `isOnE2eFrontier` + * (official: `useChartData`; overlay: `processOverlayChartData`) so the line is + * restricted to the e2e-Pareto winners — the same set of points across every + * chart, just re-plotted at the chosen x metric. When the flag is present on + * ANY point in the bucket, narrow to the winners before paretoing; otherwise + * (fixed-seq, the e2e chart itself, or metrics with no e2e roofline direction) + * use every eligible point — recomputing a fresh frontier on the swapped x + * axis would reintroduce the benchmark hack. Single interpreter of the flag's + * tri-state (undefined / true / false) for both the official and overlay + * roofline memos. + */ +export function e2eRestrictedSeed(points: InferenceData[]): InferenceData[] { + const flagged = points.some((p) => p.isOnE2eFrontier !== undefined); + return (flagged ? points.filter((p) => p.isOnE2eFrontier === true) : points).filter( + isFrontierEligible, + ); +} diff --git a/packages/app/src/components/inference/utils/resolveXAxisField.ts b/packages/app/src/components/inference/utils/resolveXAxisField.ts new file mode 100644 index 00000000..661bc558 --- /dev/null +++ b/packages/app/src/components/inference/utils/resolveXAxisField.ts @@ -0,0 +1,91 @@ +/** + * @file resolveXAxisField.ts + * @description Single source of truth for which data field a chart's x-axis + * plots. Both the official pipeline (`useChartData.stableChartDefinitions`) + * and the `?unofficialrun=` overlay pipeline (`processOverlayChartData`) call + * this — they previously each carried a copy of the branch ladder held in sync + * only by "must mirror" comments, and three of the four overlay-misalignment + * bugs fixed on this branch were exactly that mirror drifting. Change the + * resolution logic here and both paths move together. + * + * (`buildReplayTimeline.resolveXAxisField` is a third, older copy — replay is + * fixed-seq-only today so its missing percentile handling is inert, but it + * should adopt this resolver if replay ever grows agentic support.) + */ +import { withPercentile } from '@/lib/benchmark-transform'; + +import type { ChartDefinition } from '../types'; + +/** Which rung of the branch ladder chose the x field (drives label choice). */ +export type XAxisBranch = + | 'natural' + | 'user-input-override' + | 'config-input-override' + | 'e2e-ttft-override'; + +export interface ResolvedXAxis { + /** The data field the x-axis plots (percentile-adjusted for agentic). */ + xAxisField: string; + /** The chart's natural latency metric, percentile-adjusted — the "no + * override" baseline the flip check compares against. */ + naturalX: string; + isInputMetric: boolean; + isTtftOverride: boolean; + branch: XAxisBranch; +} + +/** + * Resolve the x-axis data field for a chart definition + metric selection. + * + * Rules, in order: + * - Natural x = the chart's latency metric at the selected percentile for + * agentic, forced to median for fixed-seq (whose p90_/p99_ columns don't + * exist — a stale 'p90' from a previous agentic view would resolve to a + * null column and drop every point). + * - Input metrics on the interactivity chart override x to a TTFT column: + * the user-picked metric for fixed-seq (the manual dropdown is hidden in + * agentic mode), else the config default. + * - Any *_ttft `effectiveXMetric` overrides the e2e chart's x (the 'ttft' + * x-axis mode) — the percentile prefix was already reconciled by the + * x-axis-mode picker. + * - Agentic: the resolved field is rewritten to the selected percentile. + * Idempotent for overrides that already carry it (e.g. p90_ttft), and + * carries it onto config-default overrides (median_ttft → p90_ttft) and + * the natural intvty/e2el field. + */ +export function resolveXAxisField( + chartDef: ChartDefinition, + selectedYAxisMetric: string, + effectiveXMetric: string | null, + opts: { isAgentic: boolean; percentile: string }, +): ResolvedXAxis { + const { isAgentic, percentile } = opts; + const naturalX = withPercentile(chartDef.x, isAgentic ? percentile : 'median'); + + const metricTitle = + (chartDef[`${selectedYAxisMetric}_title` as keyof ChartDefinition] as string) || ''; + const isInputMetric = metricTitle.toLowerCase().includes('input'); + // Any *_ttft metric counts — the x-axis-mode picker can select any + // percentile (median/p75/p90/p99) depending on sequence kind. + const isTtftOverride = typeof effectiveXMetric === 'string' && effectiveXMetric.endsWith('_ttft'); + + let xAxisField: string = naturalX; + let branch: XAxisBranch = 'natural'; + if (effectiveXMetric && chartDef.chartType === 'interactivity' && isInputMetric && !isAgentic) { + xAxisField = effectiveXMetric; + branch = 'user-input-override'; + } else if (chartDef.chartType === 'interactivity' && isInputMetric) { + const xOverrideKey = `${selectedYAxisMetric}_x` as keyof ChartDefinition; + xAxisField = (chartDef[xOverrideKey] as string) || chartDef.x; + branch = 'config-input-override'; + } else if (chartDef.chartType === 'e2e' && isTtftOverride) { + xAxisField = effectiveXMetric!; + branch = 'e2e-ttft-override'; + } + + if (isAgentic) { + xAxisField = withPercentile(xAxisField, percentile); + } + + return { xAxisField, naturalX, isInputMetric, isTtftOverride, branch }; +}