Skip to content

fix: align unofficial run table percentiles #605

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adibarra wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: align unofficial run table percentiles #605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/app/cypress/component/scatter-graph.cy.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -917,13 +917,17 @@ describe('ChartDisplay engine comparison guard', () => {
hw: 'Official SGLang',
model: Model.DeepSeek_V4_Pro,
precision: Precision.FP4,
p75_ttft: 0.75,
p75_intvty: 75,
});
const runUrl = 'https://github.com/x/y/actions/runs/456';
const overlayRow = createMockInferenceData({
hwKey: 'h100_vllm',
hw: 'Unofficial vLLM',
model: Model.DeepSeek_V4_Pro,
precision: Precision.FP4,
p75_ttft: 0.85,
p75_intvty: 65,
run_url: runUrl,
});
const exclusion = buildExclusion([
Expand Down Expand Up @@ -968,6 +972,7 @@ describe('ChartDisplay engine comparison guard', () => {
selectedModel: Model.DeepSeek_V4_Pro,
selectedSequence: Sequence.AgenticTraces,
selectedXAxisMode: 'interactivity',
selectedPercentile: 'p75',
activeHwTypes: new Set(['b200_sglang']),
hwTypesWithData: new Set(['b200_sglang']),
resolveComparisonSelection: resolveSelection,
Expand All @@ -992,6 +997,15 @@ describe('ChartDisplay engine comparison guard', () => {
cy.get('[data-testid="inference-results-table"] tbody tr').should('have.length', 2);
cy.get('[data-testid="inference-results-table"] tbody').contains('vLLM').should('exist');
cy.get('[data-testid="inference-results-table"] tbody').contains('SGLang').should('exist');
cy.get('[data-testid="inference-results-table"] thead')
.should('contain.text', 'P75 TTFT (ms)')
.and('contain.text', 'P75 Interactivity (tok/s)')
.and('not.contain.text', 'Median TTFT');
cy.get('[data-testid="inference-results-table"] tbody')
.should('contain.text', '750')
.and('contain.text', '850')
.and('contain.text', '75.0')
.and('contain.text', '65.0');
// The reconciliation effect must not strip the run's hw types from the provider.
cy.get('@setActiveOverlayHwTypes').should('not.have.been.called');
});
Expand Down
1 change: 1 addition & 0 deletions packages/app/src/components/inference/ui/ChartDisplay.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -871,6 +871,7 @@ export default function ChartDisplay() {
data={[...officialRows, ...overlayRows]}
chartDefinition={graph.chartDefinition}
selectedYAxisMetric={selectedYAxisMetric}
percentile={isAgenticSequence ? selectedPercentile : 'median'}
/>
</>
);
Expand Down
19 changes: 12 additions & 7 deletions packages/app/src/components/inference/ui/InferenceTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ interface InferenceTableProps {
data: InferenceData[];
chartDefinition: ChartDefinition;
selectedYAxisMetric: string;
percentile?: string;
}

/** Format a number for table display — picks sensible precision based on magnitude. */
Expand All @@ -28,10 +29,14 @@ export default function InferenceTable({
data,
chartDefinition,
selectedYAxisMetric,
percentile = 'median',
}: InferenceTableProps) {
const yPath = chartDefinition[selectedYAxisMetric as keyof ChartDefinition] as string | undefined;
const yLabel = chartDefinition[`${selectedYAxisMetric}_label` as keyof ChartDefinition] as string;
const xLabel = chartDefinition.x_label;
const percentileLabel = percentile === 'median' ? 'Median' : percentile.toUpperCase();
const ttftKey = `${percentile}_ttft` as keyof InferenceData;
const interactivityKey = `${percentile}_intvty` as keyof InferenceData;

const rooflineDir = chartDefinition[
`${selectedYAxisMetric}_roofline` as keyof ChartDefinition
Expand Down Expand Up @@ -97,21 +102,21 @@ export default function InferenceTable({
className: 'tabular-nums',
},
{
header: 'Median TTFT (ms)',
header: `${percentileLabel} TTFT (ms)`,
align: 'right',
cell: (row) => fmt((row.median_ttft ?? 0) * 1000, 0),
sortValue: (row) => row.median_ttft ?? 0,
cell: (row) => fmt(((row[ttftKey] as number | undefined) ?? 0) * 1000, 0),
sortValue: (row) => (row[ttftKey] as number | undefined) ?? 0,
className: 'tabular-nums',
},
{
header: 'Median Interactivity (tok/s)',
header: `${percentileLabel} Interactivity (tok/s)`,
align: 'right',
cell: (row) => fmt(row.median_intvty ?? 0, 1),
sortValue: (row) => row.median_intvty ?? 0,
cell: (row) => fmt((row[interactivityKey] as number | undefined) ?? 0, 1),
sortValue: (row) => (row[interactivityKey] as number | undefined) ?? 0,
className: 'tabular-nums',
},
],
[yPath, yLabel, xLabel],
[yPath, yLabel, xLabel, percentileLabel, ttftKey, interactivityKey],
);

return (
Expand Down
Loading