From 8b6b2484e5a960cf95d6a170ee7c4377bb65e47f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 29 Jul 2026 14:56:26 -0700 Subject: [PATCH 1/4] feat(overview): add scenario-aware comparisons --- packages/app/cypress/e2e/navigation.cy.ts | 12 +- packages/app/cypress/e2e/overview.cy.ts | 161 +++++++------- packages/app/cypress/e2e/zh-pages.cy.ts | 11 +- .../app/src/app/(dashboard)/overview/page.tsx | 2 +- .../src/app/zh/(dashboard)/overview/page.tsx | 2 +- .../calculator/useThroughputData.ts | 49 +---- .../src/components/landing/landing-page.tsx | 47 +++-- .../src/components/overview/overview-page.tsx | 5 +- .../overview/overview-scorecard.tsx | 167 +++++++-------- packages/app/src/lib/agentic-frontier.ts | 49 +++++ packages/app/src/lib/overview-data.test.ts | 126 ++++++++--- packages/app/src/lib/overview-data.ts | 199 ++++++++++++------ packages/app/src/lib/overview-links.test.ts | 19 ++ packages/app/src/lib/overview-links.ts | 9 +- packages/app/src/lib/tco-feed.ts | 153 +++++++------- 15 files changed, 585 insertions(+), 426 deletions(-) create mode 100644 packages/app/src/lib/agentic-frontier.ts diff --git a/packages/app/cypress/e2e/navigation.cy.ts b/packages/app/cypress/e2e/navigation.cy.ts index 63cebe8af..10bfa30a6 100644 --- a/packages/app/cypress/e2e/navigation.cy.ts +++ b/packages/app/cypress/e2e/navigation.cy.ts @@ -75,8 +75,16 @@ describe('First-load navigation', () => { cy.location('pathname').should('eq', '/compare'); }); - it('navigates to dashboard from the landing CTA with one click', () => { - cy.contains('a', 'Open Dashboard').click(); + it('navigates to overview and the full dashboard from the landing CTAs', () => { + cy.get('[data-testid="landing-overview-link"]') + .should('have.attr', 'href', '/overview') + .click(); + cy.location('pathname').should('eq', '/overview'); + + cy.visit('/'); + cy.get('[data-testid="landing-full-dashboard-link"]') + .should('have.attr', 'href', '/inference') + .click(); cy.location('pathname').should('eq', '/inference'); }); diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 903372f66..2b825679e 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -9,13 +9,8 @@ const MODEL_LABELS = [ 'Qwen3.5 397B', ]; -// Models whose row collapses to a coverage note instead of 5 platform cells: -// GLM5.2 (fixtures carry no 8K/1K rows) and Kimi K3 (no fixture rows at all — -// nothing ingested yet). -const ZERO_COVERAGE_MODELS = 2; - const PLATFORM_HEADERS = [ - 'Model', + 'Model · Scenario', 'B200 · Reference', 'MI355X', 'B300', @@ -96,17 +91,15 @@ describe('Overview page', () => { }); }); - desktopModel('GLM-5.2') - .find('[data-testid="overview-model-coverage-note"]') - .should('have.text', 'No 8K/1K results'); - desktopModel('GLM-5.2').find('[data-testid="overview-pair-missing"]').should('not.exist'); + desktopModel('GLM-5.2').within(() => { + cy.get('[data-testid="overview-model-scenario"]').should('have.text', 'AgentX'); + cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); + }); cy.get( '[data-testid="overview-engine-scope-switcher"] [data-overview-engine-scope="all"]', ).click(); cy.location('search').should('eq', '?engine=all'); - desktopModel('GLM-5.2') - .find('[data-testid="overview-model-coverage-note"]') - .should('have.text', 'No 8K/1K results'); + desktopModel('GLM-5.2').find('[data-testid="overview-pair-missing"]').should('have.length', 5); cy.get('[data-testid="overview-tier-switcher"]') .contains('a', '100') .should('have.attr', 'href', '/overview?tier=100&engine=all') @@ -134,42 +127,30 @@ describe('Overview page', () => { }); }); - it('shows cost per Mtok with standard-decode and FP8 labels', () => { + it('shows only speculative-decode reads, with FP8 and method labels', () => { cy.viewport(1280, 900); cy.visit('/overview'); desktopModel('Kimi-K2.5').within(() => { - platform('b200') - .should('contain.text', '$1.01') - .and('contain.text', 'SGLang · FP4 · Standard decode'); - platform('mi355x').within(() => { - cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"]') - .should('have.text', '$0.58') - .and('not.contain.text', '≈') - .and('not.have.attr', 'title'); - cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"] a').should('not.exist'); - cy.get('[data-testid="overview-pair-evidence-date"][data-hardware="mi355x"]') - .find('a') - .should('have.attr', 'href') - .and('include', 'i_spec=stp') - .and('include', 'i_metric=y_outputTputPerGpu'); - cy.get('[data-testid="overview-pair-evidence-date"][data-hardware="mi355x"]') - .find('a') - .should('have.attr', 'aria-label') - .and('include', 'Open raw source dashboard'); - cy.contains('SGLang · FP4 · Standard decode').should('exist'); - }); - platform('b300').within(() => { - cy.get('[data-testid="overview-pair-value"][data-hardware="b300"]').should( - 'contain.text', - '$1.07', - ); - cy.contains('SGLang · FP8 · Standard decode').should('exist'); - }); - cy.get('a[title*="NONE"], a[aria-label*="NONE"]').should('not.exist'); + cy.get('[data-testid="overview-pair-missing"]') + .should('have.length', 5) + .then(($missing) => { + expect([...$missing].map((element) => element.getAttribute('title'))).to.deep.equal([ + 'no speculative-decode result', + 'no speculative-decode result', + 'no speculative-decode result', + 'no data for this scenario', + 'no data for this scenario', + ]); + }); + cy.get('[data-testid="overview-pair-missing"]') + .children('[aria-hidden="true"]') + .should('have.length', 5) + .and('have.text', '∞∞∞∞∞'); }); desktopModel('DeepSeek-V4-Pro').within(() => { + cy.contains('MTP').should('exist'); platform('gb300').within(() => { cy.get('[data-testid="overview-pair-value"][data-hardware="gb300"]') .should('have.attr', 'title', 'Estimated from validated benchmark runs.') @@ -183,28 +164,21 @@ describe('Overview page', () => { cy.get('[data-testid="overview-pair-missing"]').should('not.exist'); }); }); + cy.get('body').should('not.contain.text', 'Standard decode'); + cy.get('body').should('not.contain.text', 'P90'); }); it('color-grades the cost delta against B200 and omits it without a baseline', () => { cy.viewport(1280, 900); cy.visit('/overview'); - desktopModel('Kimi-K2.5').within(() => { + desktopModel('Qwen-3.5-397B-A17B').within(() => { platform('b200').find('[data-testid="overview-cost-delta"]').should('not.exist'); platform('mi355x') .find('[data-testid="overview-cost-delta"]') - .should('have.text', '-42%42% cheaper than B200') .and('have.attr', 'data-cost-polarity', 'cheaper') - .and('have.attr', 'title', '42% cheaper than B200') - .then(($badge) => { - expect($badge.attr('style')).to.contain('rgb(16 185 129 / 0.35)'); - }); - platform('b300') - .find('[data-testid="overview-cost-delta"]') - .should('contain.text', '+7%') - .and('have.attr', 'data-cost-polarity', 'pricier') .then(($badge) => { - expect($badge.attr('style')).to.contain('rgb(239 68 68 / 0.12)'); + expect($badge.attr('style')).to.contain('rgb(16 185 129 /'); }); }); @@ -238,11 +212,14 @@ describe('Overview page', () => { 'Every active model across MI355X, B200, B300, GB200 and GB300 at a glance.', ).should('exist'); cy.contains( - 'Cost per million output tokens from the best observed platform serving envelopes', - ).should('exist'); - cy.contains( - 'GPU rental cost / 1M output tokens · 8K→1K · @50 tok/s/user · ↓ lower is better', + 'Cost per million output tokens from each platform’s best observed serving envelope', ).should('exist'); + cy.get('[data-testid="overview-scope"]') + .should( + 'have.text', + 'GPU rental cost / 1M output tokens · @50 tok/s/user · ↓ lower is better', + ) + .and('not.contain.text', '8K→1K'); cy.contains( 'Cost = 3-yr rental $/GPU/hr ÷ output tok/s per deployed GPU. All percentages compare against B200.', ).should('exist'); @@ -269,17 +246,24 @@ describe('Overview page', () => { ); }); cy.get('[data-testid="overview-desktop-model"]').should('have.length', MODEL_LABELS.length); - // Zero-coverage rows collapse to a single note instead of 5 cells. - cy.get('[data-testid="overview-platform"]').should( - 'have.length', - (MODEL_LABELS.length - ZERO_COVERAGE_MODELS) * 5, - ); + cy.get('[data-testid="overview-platform"]').should('have.length', MODEL_LABELS.length * 5); + cy.get('[data-testid="overview-model-coverage-note"]').should('not.exist'); cy.get('details, summary, button').should('not.exist'); cy.contains(/PRIMARY|Ranked results/).should('not.exist'); }); for (const label of MODEL_LABELS) { cy.get('[data-testid="overview-desktop-matrix"]').should('contain.text', label); } + for (const model of ['Kimi-K3', 'GLM-5.2']) { + desktopModel(model) + .find('[data-testid="overview-model-scenario"]') + .should('have.text', 'AgentX'); + } + for (const model of ['DeepSeek-V4-Pro', 'Kimi-K2.5', 'MiniMax-M3', 'Qwen-3.5-397B-A17B']) { + desktopModel(model) + .find('[data-testid="overview-model-scenario"]') + .should('have.text', 'Single-turn · 8K→1K'); + } }); it('shows per-cell best reads with precision badges, dates, and evidence links', () => { @@ -344,7 +328,7 @@ describe('Overview page', () => { platform('b200') .find('[data-testid="overview-pair-missing"]') .should('contain.text', '∞') - .and('have.attr', 'title', 'no 8K/1K data'); + .and('have.attr', 'title', 'no data for this scenario'); platform('gb300').within(() => { cy.get('[data-testid="overview-pair-value"][data-hardware="gb300"]').should( 'contain.text', @@ -353,9 +337,10 @@ describe('Overview page', () => { }); }); - desktopModel('GLM-5.2') - .find('[data-testid="overview-model-coverage-note"]') - .should('have.text', 'No 8K/1K results'); + desktopModel('GLM-5.2').within(() => { + cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); + cy.get('[data-testid="overview-model-coverage-note"]').should('not.exist'); + }); cy.contains('∞ = no comparable result').should('exist'); cy.get('body') .invoke('text') @@ -374,9 +359,10 @@ describe('Overview page', () => { }); cy.location('search').should('eq', '?tier=100'); - cy.contains( - 'GPU rental cost / 1M output tokens · 8K→1K · @100 tok/s/user · ↓ lower is better', - ).should('exist'); + cy.get('[data-testid="overview-scope"]').should( + 'have.text', + 'GPU rental cost / 1M output tokens · @100 tok/s/user · ↓ lower is better', + ); cy.get('[data-testid="overview-tier-switcher"]').within(() => { cy.get('[aria-current="page"]').should('have.text', '100'); cy.contains('a', '50').should('have.attr', 'href', '/overview'); @@ -414,17 +400,16 @@ describe('Overview page', () => { .should('not.match', /∞\s*%/); cy.visit('/overview?tier=100'); - cy.contains( - 'GPU rental cost / 1M output tokens · 8K→1K · @100 tok/s/user · ↓ lower is better', - ).should('exist'); + cy.get('[data-testid="overview-scope"]').should( + 'have.text', + 'GPU rental cost / 1M output tokens · @100 tok/s/user · ↓ lower is better', + ); cy.get('[data-testid="language-toggle"]') .should('have.attr', 'href', '/zh/overview?tier=100') .click(); cy.location('pathname').should('eq', '/zh/overview'); cy.location('search').should('eq', '?tier=100'); - cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @100 tok/s/用户 · ↓ 越低越好').should( - 'exist', - ); + cy.contains('GPU 租赁成本 / 每百万输出 token · @100 tok/s/用户 · ↓ 越低越好').should('exist'); }); it('uses the same cell semantics on mobile and fits both 390px and 320px widths', () => { @@ -551,7 +536,7 @@ describe('Overview page', () => { cy.viewport(390, 844); cy.visit('/overview'); - mobileModel('Kimi-K2.5').within(() => { + mobileModel('Qwen-3.5-397B-A17B').within(() => { platform('mi355x').within(() => { cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"]').then(([value]) => { cy.get('[data-testid="overview-pair-evidence-date"][data-hardware="mi355x"]').then( @@ -565,7 +550,7 @@ describe('Overview page', () => { }); cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"]').then(([value]) => { - cy.contains('div', 'SGLang · FP4 · Standard decode').then(([metadata]) => { + cy.contains('div', 'SGLang · FP8 · MTP').then(([metadata]) => { const valueRect = textRect(value); const metadataRect = textRect(metadata); @@ -618,10 +603,13 @@ describe('Overview page', () => { cy.contains('一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。').should( 'exist', ); - cy.contains('基于最佳观测平台服务包络线计算的各活跃模型每百万输出 token 成本').should('exist'); - cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @50 tok/s/用户 · ↓ 越低越好').should( + cy.contains('按各模型标注的场景，基于各平台最佳观测服务包络线计算每百万输出 token 成本').should( 'exist', ); + cy.get('[data-testid="overview-scope"]').should( + 'have.text', + 'GPU 租赁成本 / 每百万输出 token · @50 tok/s/用户 · ↓ 越低越好', + ); cy.contains('成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。').should('exist'); cy.contains('分离式结果的分母同时计入预填充与解码 GPU。').should('exist'); cy.contains( @@ -644,19 +632,18 @@ describe('Overview page', () => { .invoke('text') .should('not.match', /100 档由.+领先/); desktopModel('Kimi-K2.5').within(() => { - platform('mi355x').should('contain.text', 'SGLang · FP4 · 标准解码'); - platform('b300').should('contain.text', 'SGLang · FP8 · 标准解码'); + cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); platform('b200').find('[data-testid="overview-cost-delta"]').should('not.exist'); }); - desktopModel('GLM-5.2') - .find('[data-testid="overview-model-coverage-note"]') - .should('have.text', '暂无 8K/1K 结果'); + desktopModel('GLM-5.2').within(() => { + cy.get('[data-testid="overview-model-scenario"]').should('have.text', 'AgentX'); + cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); + }); + cy.get('body').should('not.contain.text', '标准解码'); cy.contains('∞ = 无可比结果').should('exist'); cy.visit('/zh/overview?tier=100'); - cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @100 tok/s/用户 · ↓ 越低越好').should( - 'exist', - ); + cy.contains('GPU 租赁成本 / 每百万输出 token · @100 tok/s/用户 · ↓ 越低越好').should('exist'); cy.get('[data-testid="overview-tier-switcher"]').within(() => { cy.contains('a', '50').should('have.attr', 'href', '/zh/overview'); }); diff --git a/packages/app/cypress/e2e/zh-pages.cy.ts b/packages/app/cypress/e2e/zh-pages.cy.ts index 77fc8cfc5..70c0a6a85 100644 --- a/packages/app/cypress/e2e/zh-pages.cy.ts +++ b/packages/app/cypress/e2e/zh-pages.cy.ts @@ -5,12 +5,17 @@ describe('Chinese (/zh) pages', () => { }); it('renders the Chinese landing content', () => { - cy.contains('h2', '完整仪表板').should('exist'); + cy.contains('h2', '探索 InferenceX').should('exist'); cy.contains('快速对比').should('exist'); }); - it('links into the Chinese dashboard tree', () => { - cy.get('a[href="/zh/inference"]').should('exist'); + it('links to the Chinese overview and full dashboard', () => { + cy.get('[data-testid="landing-overview-link"]') + .should('have.attr', 'href', '/zh/overview') + .and('have.text', '总览'); + cy.get('[data-testid="landing-full-dashboard-link"]') + .should('have.attr', 'href', '/zh/inference') + .and('have.text', '完整仪表板'); }); it('sets hreflang alternates to the English homepage', () => { diff --git a/packages/app/src/app/(dashboard)/overview/page.tsx b/packages/app/src/app/(dashboard)/overview/page.tsx index c324a26fe..564c3f7ce 100644 --- a/packages/app/src/app/(dashboard)/overview/page.tsx +++ b/packages/app/src/app/(dashboard)/overview/page.tsx @@ -10,7 +10,7 @@ import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; const DESCRIPTION = - 'Cost per million output tokens from the best observed platform serving envelopes across MI355X, B200, B300, GB200 and GB300 at a fixed single-turn 8K input / 1K output workload, prioritizing speculative decode and FP4.'; + 'Compare cost per million output tokens across MI355X, B200, B300, GB200 and GB300 using the scenario shown for each active model.'; export const metadata: Metadata = { title: 'Inference Cost Overview', diff --git a/packages/app/src/app/zh/(dashboard)/overview/page.tsx b/packages/app/src/app/zh/(dashboard)/overview/page.tsx index 5ddeb6fd3..e3aaf44a6 100644 --- a/packages/app/src/app/zh/(dashboard)/overview/page.tsx +++ b/packages/app/src/app/zh/(dashboard)/overview/page.tsx @@ -10,7 +10,7 @@ import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; const DESCRIPTION = - '在固定单轮 8K 输入 / 1K 输出负载下，基于最佳观测平台服务包络线对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的每百万输出 token 成本；优先采用推测解码与 FP4。'; + '按各活跃模型标注的场景，对比 MI355X、B200、B300、GB200 与 GB300 的每百万输出 token 成本。'; export const metadata: Metadata = { title: '推理成本总览', diff --git a/packages/app/src/components/calculator/useThroughputData.ts b/packages/app/src/components/calculator/useThroughputData.ts index 01ce8bfca..facd79d2b 100644 --- a/packages/app/src/components/calculator/useThroughputData.ts +++ b/packages/app/src/components/calculator/useThroughputData.ts @@ -4,11 +4,11 @@ import { useCallback, useMemo } from 'react'; import { DB_MODEL_TO_DISPLAY, rowToSequence } from '@semianalysisai/inferencex-constants'; -import type { AggDataEntry, HardwareConfig, InferenceData } from '@/components/inference/types'; +import type { AggDataEntry, HardwareConfig } from '@/components/inference/types'; import { useBenchmarks } from '@/hooks/api/use-benchmarks'; import type { BenchmarkRow } from '@/lib/api'; import { rowToAggDataEntry } from '@/lib/benchmark-transform'; -import { getHardwareKey, paretoFrontUpperRight } from '@/lib/chart-utils'; +import { getHardwareKey } from '@/lib/chart-utils'; import { getModelSortIndex, getHardwareConfig, getGpuSpecs } from '@/lib/constants'; import { Percentile, Sequence, type Model } from '@/lib/data-mappings'; import { overlayRunIndex } from '@/lib/overlay-run-style'; @@ -22,6 +22,7 @@ import { paretoFrontUpperLeft, sign, } from './interpolation'; +import { restrictAgenticPointsToE2eFrontier } from '@/lib/agentic-frontier'; import type { CostProvider, GPUDataPoint, InterpolatedResult } from './types'; // Re-export pure functions so existing imports from this module keep working. @@ -52,50 +53,6 @@ export interface OverlayGroupMeta extends GroupMeta { runIndex: number; } -interface AgenticFrontierPoint { - x: number; - y: number; - orig: GPUDataPoint; -} - -/** - * Match the main agentic interactivity chart's anti-benchmark-hacking seed: - * within each date, only points that also win on the (end-to-end latency, - * total throughput) upper-right Pareto frontier may feed interpolation. - * - * The chart's shared Pareto helper operates on `InferenceData.x/y` only, so - * minimal projections keep this path aligned without running the full chart - * transformation pipeline. - */ -export function restrictAgenticPointsToE2eFrontier(points: GPUDataPoint[]): GPUDataPoint[] { - const byDate = new Map(); - - for (const point of points) { - if ( - typeof point.e2eLatency !== 'number' || - !Number.isFinite(point.e2eLatency) || - !Number.isFinite(point.throughput) - ) { - continue; - } - const date = point.date ?? ''; - let bucket = byDate.get(date); - if (!bucket) { - bucket = []; - byDate.set(date, bucket); - } - bucket.push({ x: point.e2eLatency, y: point.throughput, orig: point }); - } - - const winners = new Set(); - for (const bucket of byDate.values()) { - for (const winner of paretoFrontUpperRight(bucket as unknown as InferenceData[])) { - winners.add((winner as unknown as AgenticFrontierPoint).orig); - } - } - return points.filter((point) => winners.has(point)); -} - function getAgenticMetric( entry: AggDataEntry, percentile: Percentile, diff --git a/packages/app/src/components/landing/landing-page.tsx b/packages/app/src/components/landing/landing-page.tsx index 61c4f0d76..02f10f294 100644 --- a/packages/app/src/components/landing/landing-page.tsx +++ b/packages/app/src/components/landing/landing-page.tsx @@ -11,12 +11,13 @@ import type { Locale } from '@/lib/i18n'; const STRINGS = { en: { + exploreInferenceX: 'Explore InferenceX', + exploreInferenceXLead: + 'Start with a concise cost overview across active models and key platforms, or open the full dashboard for every model, GPU, framework, and metric.', fullDashboard: 'Full Dashboard', - fullDashboardP1: - 'Every model, GPU, framework, and metric. Fully configurable inference benchmark charts with date ranges, concurrency sweeps, and raw data export.', - fullDashboardP2: + platformCoverage: 'Compare NVIDIA GB300 NVL72, GB200 NVL72, B300, B200, H200, H100, AMD MI355X, MI325X, MI300X and soon VR200 NVL72, AMD MI455X UALoE72, TPUv7 Ironwood, etc across DeepSeekv4 Pro, Qwen, Kimi, GLM, MiniMax, gpt-oss, Llama and other models.', - openDashboard: 'Open Dashboard', + overview: 'Overview', reproTitle: 'Every Result Is Transparently done through Public GitHub Actions Automation', reproP1: 'Every data point on the dashboard is produced by a public GitHub Actions workflow run. The recipe lives in the repo, the run executes on the actual target hardware, and the full logs and artifacts are publicly viewable. Click any point on a chart to jump straight to the run that produced it. All reproducible, auditable, and open source.', @@ -39,12 +40,13 @@ const STRINGS = { 'Jump straight into the most popular GPU inference benchmark comparisons, curated and ready to explore.', }, zh: { + exploreInferenceX: '探索 InferenceX', + exploreInferenceXLead: + '先查看活跃模型与关键平台的精简成本总览，或打开完整仪表板，探索所有模型、GPU、框架与指标。', fullDashboard: '完整仪表板', - fullDashboardP1: - '覆盖所有模型、GPU、框架与指标。完全可配置的推理基准测试图表，支持日期范围、并发扫描与原始数据导出。', - fullDashboardP2: + platformCoverage: '跨 DeepSeekv4 Pro、Qwen、Kimi、GLM、MiniMax、gpt-oss、Llama 等模型，对比 NVIDIA GB300 NVL72、GB200 NVL72、B300、B200、H200、H100、AMD MI355X、MI325X、MI300X，以及即将上线的 VR200 NVL72、AMD MI455X UALoE72、TPUv7 Ironwood 等硬件。', - openDashboard: '打开仪表板', + overview: '总览', reproTitle: '每一条结果都通过公开的 GitHub Actions 自动化流程透明产生', reproP1: '仪表板上的每个数据点都由公开的 GitHub Actions 工作流运行产生。配置方案（recipe）保存在公开仓库中，运行在真实目标硬件上执行，完整日志与产物公开可查。点击图表上的任意数据点即可跳转到生成它的那次运行。一切都可复现、可审计、开源。', @@ -76,25 +78,36 @@ export function LandingPage({ locale = 'en' }: { locale?: Locale } = {}) {
- {/* Split: Dashboard vs Presets */} + {/* Split: exploration entry points vs presets */}
- {/* Left - Full Dashboard */} + {/* Primary product entry points */}
-

{t.fullDashboard}

+

{t.exploreInferenceX}

-

{t.fullDashboardP1}

-

{t.fullDashboardP2}

-
+

{t.exploreInferenceXLead}

+

{t.platformCoverage}

+
+ + {t.overview} + - {t.openDashboard} - + {t.fullDashboard} +
diff --git a/packages/app/src/components/overview/overview-page.tsx b/packages/app/src/components/overview/overview-page.tsx index a346ae763..2728535bd 100644 --- a/packages/app/src/components/overview/overview-page.tsx +++ b/packages/app/src/components/overview/overview-page.tsx @@ -31,7 +31,10 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) {

{strings.title}

{strings.purpose}

-

+

{strings.scope(data.tier)}

{snapshot === null ? null : ( diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index 8c3574c83..732563109 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -21,7 +21,7 @@ export const OVERVIEW_STRINGS = { title: 'Inference Cost Overview', purpose: 'Every active model across MI355X, B200, B300, GB200 and GB300 at a glance.', scope: (tier: number) => - `GPU rental cost / 1M output tokens · 8K→1K · @${tier} tok/s/user · ↓ lower is better`, + `GPU rental cost / 1M output tokens · @${tier} tok/s/user · ↓ lower is better`, tierNavLabel: 'Service level', tierUnit: 'tok/s/user', engineScopeNavLabel: 'Engine scope', @@ -31,28 +31,31 @@ export const OVERVIEW_STRINGS = { }, snapshot: (through: string) => `Database snapshot through ${through}`, caption: - "Cost per million output tokens from the best observed platform serving envelopes for every active model across today's key platforms, prioritizing speculative decode and FP4.", - modelHeader: 'Model', + 'Cost per million output tokens from each platform’s best observed serving envelope for the scenario shown with each model.', + modelHeader: 'Model · Scenario', + scenarioLabels: { + single_turn_8k1k: 'Single-turn · 8K→1K', + agentx: 'AgentX', + }, detailsHeader: 'Details', detailLink: 'View details', detailAria: (modelLabel: string) => `View details: ${modelLabel}`, rawDashboardAria: (evidenceDate: string, modelLabel: string, stack: string) => `Open raw source dashboard for ${evidenceDate}: ${modelLabel} · ${stack}`, - standardDecode: 'Standard decode', estimatedTooltip: (topologies: readonly string[]) => topologies.length === 0 ? 'Estimated from validated benchmark runs.' : `Estimated from validated ${topologies.join(' and ')} runs.`, estimatedAria: (value: string, explanation: string) => `Approximately ${value}. ${explanation}`, - noWorkloadResults: 'No 8K/1K results', infinityLegend: '∞ = no comparable result', missingReasons: (tier: number): Record => ({ int4_bf16_only: 'INT4/BF16 only', - no_8k1k_data: 'no 8K/1K data', + no_scenario_data: 'no data for this scenario', + no_speculative_decode_result: 'no speculative-decode result', cannot_reach_at_tier: `cannot reach @${tier}`, no_exact_at_tier: `no exact @${tier} result`, }), - methodologyNote: 'Priority: speculative FP4 → speculative FP8 → standard FP4 → standard FP8.', + methodologyNote: 'Speculative decode only. Priority: FP4 → FP8.', costNote: 'Cost = 3-yr rental $/GPU/hr ÷ output tok/s per deployed GPU. All percentages compare against B200.', costDeltaAria: (pct: string, cheaper: boolean) => @@ -64,13 +67,12 @@ export const OVERVIEW_STRINGS = { interpolationNote: 'Tier values use the best observed platform serving envelope; ≈ marks estimates between validated runs. No extrapolation.', comparabilityNote: - 'Directional platform comparison: cells pick each platform’s best observed envelope, so dates, engines, precisions and decode methods may differ.', + 'Each row compares platforms within the scenario shown with that model; dates, engines, precisions and speculative methods may differ.', }, zh: { title: '推理成本总览', purpose: '一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。', - scope: (tier: number) => - `GPU 租赁成本 / 每百万输出 token · 8K→1K · @${tier} tok/s/用户 · ↓ 越低越好`, + scope: (tier: number) => `GPU 租赁成本 / 每百万输出 token · @${tier} tok/s/用户 · ↓ 越低越好`, tierNavLabel: '服务档位', tierUnit: 'tok/s/用户', engineScopeNavLabel: '引擎范围', @@ -79,29 +81,31 @@ export const OVERVIEW_STRINGS = { community: '开源社区引擎（vLLM/SGLang）', }, snapshot: (through: string) => `数据库快照截至 ${through}`, - caption: - '基于最佳观测平台服务包络线计算的各活跃模型每百万输出 token 成本；优先采用推测解码与 FP4。', - modelHeader: '模型', + caption: '按各模型标注的场景，基于各平台最佳观测服务包络线计算每百万输出 token 成本。', + modelHeader: '模型 · 场景', + scenarioLabels: { + single_turn_8k1k: '单轮 · 8K→1K', + agentx: 'AgentX', + }, detailsHeader: '详情', detailLink: '查看详情', detailAria: (modelLabel: string) => `查看详情：${modelLabel}`, rawDashboardAria: (evidenceDate: string, modelLabel: string, stack: string) => `打开 ${evidenceDate} 原始数据仪表板：${modelLabel} · ${stack}`, - standardDecode: '标准解码', estimatedTooltip: (topologies: readonly string[]) => topologies.length === 0 ? '根据已验证的基准运行结果估算。' : `根据已验证的 ${topologies.join(' 与 ')} 运行结果估算。`, estimatedAria: (value: string, explanation: string) => `约 ${value}。${explanation}`, - noWorkloadResults: '暂无 8K/1K 结果', infinityLegend: '∞ = 无可比结果', missingReasons: (tier: number): Record => ({ int4_bf16_only: '仅 INT4/BF16', - no_8k1k_data: '无 8K/1K 数据', + no_scenario_data: '该场景暂无数据', + no_speculative_decode_result: '暂无推测解码结果', cannot_reach_at_tier: `无法达到 @${tier}`, no_exact_at_tier: `无精确 @${tier} 结果`, }), - methodologyNote: '优先顺序：推测解码 FP4 → 推测解码 FP8 → 标准解码 FP4 → 标准解码 FP8。', + methodologyNote: '仅展示推测解码。优先顺序：FP4 → FP8。', costNote: '成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。所有百分比均相对 B200。', costDeltaAria: (pct: string, cheaper: boolean) => `比 B200 ${cheaper ? '便宜' : '昂贵'} ${pct}`, costDeltaEvenAria: '与 B200 成本基本持平', @@ -109,8 +113,7 @@ export const OVERVIEW_STRINGS = { normalizationNote: '分离式结果的分母同时计入预填充与解码 GPU。', interpolationNote: '各档位数值采用最佳观测平台服务包络线；≈ 表示根据已验证运行结果估算。不会外推。', - comparabilityNote: - '方向性平台对比：各单元格取该平台最佳观测包络线，日期、引擎、精度与解码方式可能不同。', + comparabilityNote: '每行均在该模型标注的场景内比较各平台；日期、引擎、精度与推测方法可能不同。', }, } as const; @@ -266,11 +269,7 @@ function CellValue({ const stackBadge = config === null || precisionLabel === null ? null - : [ - config.frameworkLabel, - precisionLabel, - member.decodeMode === 'standard' ? strings.standardDecode : null, - ] + : [config.frameworkLabel, precisionLabel, config.specLabel] .filter((part): part is string => part !== null) .join(' · '); const stack = @@ -280,7 +279,7 @@ function CellValue({ member.hardwareLabel, config.frameworkLabel, config.precision.toUpperCase(), - member.decodeMode === 'standard' ? strings.standardDecode : config.specLabel, + config.specLabel, ].join(' · '); const evidenceDateLabel = evidenceDate === null ? '' : formatEvidenceDate(formatters, evidenceDate); @@ -382,27 +381,17 @@ function PlatformCell(props: { ); } -/** A model with zero 8K/1K coverage collapses to one note instead of a row of - * identical empty states. */ -function hasNo8k1kResult(model: OverviewModelSummary): boolean { +function ModelName({ model, strings }: { model: OverviewModelSummary; strings: OverviewStrings }) { return ( - model.platforms.length > 0 && - model.platforms.every((platform) => platform.missingReason === 'no_8k1k_data') - ); -} - -function ModelName({ model }: { model: OverviewModelSummary }) { - return

{model.modelLabel}

; -} - -function CoverageNote({ strings }: { strings: OverviewStrings }) { - return ( -

- {strings.noWorkloadResults} -

+
+

{model.modelLabel}

+

+ {strings.scenarioLabels[model.scenario]} +

+
); } @@ -456,28 +445,22 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S className="border-b border-border/50 align-top last:border-b-0" > - + - {hasNo8k1kResult(model) ? ( - - + {model.platforms.map((platform) => ( + + - ) : ( - model.platforms.map((platform) => ( - - - - )) - )} + ))} - - {hasNo8k1kResult(model) ? ( - - ) : ( -
- {model.platforms.map((platform) => ( -
+
+ {model.platforms.map((platform) => ( +
+ - - {platform.hardwareLabel} - - -
- ))} -
- )} + {platform.hardwareLabel} + + +
+ ))} +
{ + x: number; + y: number; + orig: T; +} + +/** + * Keep only date-scoped winners on the AgentX + * (E2E latency, selected throughput metric) Pareto frontier. + */ +export function restrictAgenticPointsToE2eFrontier( + points: readonly T[], +): T[] { + const byDate = new Map[]>(); + + for (const point of points) { + if ( + !Number.isFinite(point.e2eLatency) || + (point.e2eLatency ?? 0) <= 0 || + !Number.isFinite(point.throughput) || + point.throughput <= 0 + ) { + continue; + } + const date = point.date ?? ''; + const framed = { x: point.e2eLatency!, y: point.throughput, orig: point }; + const bucket = byDate.get(date); + if (bucket) bucket.push(framed); + else byDate.set(date, [framed]); + } + + const winners = new Set(); + for (const bucket of byDate.values()) { + for (const winner of paretoFrontUpperRight(bucket as unknown as InferenceData[])) { + winners.add((winner as unknown as FramedPoint).orig); + } + } + return points.filter((point) => winners.has(point)); +} diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index a77a3c90f..3524fdccc 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -9,6 +9,7 @@ import { DEFAULT_MODELS, Model, Precision } from './data-mappings'; import { assembleOverviewPageData, buildOverviewModelSummary, + overviewScenarioForModel, resolveOverviewEngineScope, resolveOverviewTier, type OverviewModelSummary, @@ -84,7 +85,16 @@ function headlinePairOf(summary: OverviewModelSummary, id: string) { return candidate === undefined || baseline === undefined ? undefined : { candidate, baseline }; } -describe('overview engine scope and comparable fallbacks', () => { +describe('overview engine scope and scenario selection', () => { + it('assigns each active model to its configured scenario', () => { + expect(overviewScenarioForModel(Model.Kimi_K3)).toBe('agentx'); + expect(overviewScenarioForModel(Model.GLM_5_2)).toBe('agentx'); + expect(overviewScenarioForModel(Model.DeepSeek_V4_Pro)).toBe('single_turn_8k1k'); + expect(overviewScenarioForModel(Model.Kimi_K2_5)).toBe('single_turn_8k1k'); + expect(overviewScenarioForModel(Model.MiniMax_M3)).toBe('single_turn_8k1k'); + expect(overviewScenarioForModel(Model.Qwen3_5)).toBe('single_turn_8k1k'); + }); + it('resolves valid engine scopes and defaults invalid values to community', () => { expect(resolveOverviewEngineScope('community')).toBe('community'); expect(resolveOverviewEngineScope('all')).toBe('all'); @@ -410,7 +420,7 @@ describe('overview engine scope and comparable fallbacks', () => { expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read.value).toBeNull(); }); - it('applies the decode and precision priority ladder independently per platform', () => { + it('uses speculative FP4 then speculative FP8 without standard-decode fallback', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([900, 700, 500, 300], { hardware: 'b200', precision: Precision.FP4 }), ...frontier([1400, 1200, 1000, 800], { @@ -447,18 +457,17 @@ describe('overview engine scope and comparable fallbacks', () => { ]); expect( - summary.platforms.map(({ hardware, precision, decodeMode, read }) => ({ + summary.platforms.map(({ hardware, precision, read }) => ({ hardware, precision, - decodeMode, value: read.value, })), ).toEqual([ - { hardware: 'b200', precision: Precision.FP4, decodeMode: 'speculative', value: 700 }, - { hardware: 'mi355x', precision: Precision.FP8, decodeMode: 'speculative', value: 900 }, - { hardware: 'b300', precision: Precision.FP4, decodeMode: 'standard', value: 1100 }, - { hardware: 'gb200', precision: Precision.FP8, decodeMode: 'standard', value: 1000 }, - { hardware: 'gb300', precision: Precision.FP4, decodeMode: 'standard', value: 900 }, + { hardware: 'b200', precision: Precision.FP4, value: 700 }, + { hardware: 'mi355x', precision: Precision.FP8, value: 900 }, + { hardware: 'b300', precision: null, value: null }, + { hardware: 'gb200', precision: null, value: null }, + { hardware: 'gb300', precision: null, value: null }, ]); }); }); @@ -614,7 +623,7 @@ describe('overview platform selection', () => { ); }); - it('uses standard decode when available and still flags unsupported precision coverage', () => { + it('shows standard-only data as missing and still flags unsupported precision coverage', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), row({ hardware: 'mi355x', precision: Precision.FP8, spec_method: 'none' }), @@ -622,13 +631,72 @@ describe('overview platform selection', () => { ]); expect(headlinePairOf(summary, 'mi355x-vs-b200')?.candidate).toMatchObject({ - decodeMode: 'standard', - missingReason: null, - read: { value: 1000 }, + missingReason: 'no_speculative_decode_result', + read: { value: null }, }); expect(headlinePairOf(summary, 'b300-vs-b200')?.candidate.missingReason).toBe('int4_bf16_only'); }); + it('reads AgentX at the chart-default P90 contract without exposing P90 as the scenario', () => { + const summary = buildOverviewModelSummary(Model.GLM_5_2, [ + row({ + model: 'glm5.2', + hardware: 'b200', + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + precision: Precision.FP4, + spec_method: 'mtp', + conc: 8, + metrics: { + p90_itl: 1 / 40, + p90_ttlt: 30, + tput_per_gpu: 1400, + output_tput_per_gpu: 1200, + }, + }), + row({ + model: 'glm5.2', + hardware: 'b200', + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + precision: Precision.FP4, + spec_method: 'mtp', + conc: 12, + metrics: { + p90_itl: 1 / 50, + p90_ttlt: 25, + tput_per_gpu: 900, + output_tput_per_gpu: 850, + }, + }), + row({ + model: 'glm5.2', + hardware: 'b200', + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + precision: Precision.FP4, + spec_method: 'mtp', + conc: 16, + metrics: { + p90_itl: 1 / 60, + p90_ttlt: 20, + tput_per_gpu: 1000, + output_tput_per_gpu: 800, + }, + }), + ]); + + expect(summary.scenario).toBe('agentx'); + expect(summary.platforms.find(({ hardware }) => hardware === 'b200')?.read).toMatchObject({ + value: 850, + boundary: 'interpolated', + estimated: false, + }); + }); + it('returns all five platforms with coverage gaps for an empty model', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, []); @@ -640,9 +708,9 @@ describe('overview platform selection', () => { 'gb300', ]); expect(summary.platforms.every(({ precision }) => precision === null)).toBe(true); - expect(summary.platforms.every(({ missingReason }) => missingReason === 'no_8k1k_data')).toBe( - true, - ); + expect( + summary.platforms.every(({ missingReason }) => missingReason === 'no_scenario_data'), + ).toBe(true); }); }); @@ -774,7 +842,7 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { // MiniMax: the platform result remains visible when B200 has no 8K/1K data. const minimax = page.models.find((m) => m.model === Model.MiniMax_M3)!; const mmGb300 = headlinePairOf(minimax, 'gb300-vs-b200')!; - expect(mmGb300.baseline.missingReason).toBe('no_8k1k_data'); + expect(mmGb300.baseline.missingReason).toBe('no_scenario_data'); expect(mmGb300.candidate.read.value).toBe(700); // Qwen: MI355X independently falls back to FP8 while B200 and B300 use FP4. @@ -790,24 +858,24 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { expect(qwenB300.baseline.precision).toBe(Precision.FP4); expect(qwenB300.baseline.read.value).toBeCloseTo(733.594); - // Kimi: standard decode supplies each platform's highest-priority precision. + // Kimi: standard-only rows remain visible as spec-decode coverage gaps. const kimi = page.models.find((m) => m.model === Model.Kimi_K2_5)!; const kimiMi = headlinePairOf(kimi, 'mi355x-vs-b200')!; - expect(kimiMi.candidate.precision).toBe(Precision.FP4); - expect(kimiMi.candidate.read.value).toBe(1000); - expect(kimiMi.baseline.precision).toBe(Precision.FP4); - expect(kimiMi.baseline.read.value).toBe(800); + expect(kimiMi.candidate.precision).toBeNull(); + expect(kimiMi.candidate.read.value).toBeNull(); + expect(kimiMi.candidate.missingReason).toBe('no_speculative_decode_result'); + expect(kimiMi.baseline.precision).toBeNull(); + expect(kimiMi.baseline.read.value).toBeNull(); const kimiB300 = headlinePairOf(kimi, 'b300-vs-b200')!; - expect(kimiB300.candidate.precision).toBe(Precision.FP8); - expect(kimiB300.candidate.read.value).toBe(900); - expect(kimiB300.baseline.precision).toBe(Precision.FP4); - expect(kimiB300.baseline.read.value).toBe(800); + expect(kimiB300.candidate.precision).toBeNull(); + expect(kimiB300.candidate.read.value).toBeNull(); + expect(kimiB300.candidate.missingReason).toBe('no_speculative_decode_result'); - // GLM mirrors the live coverage gap: agentic data exists, but no single-turn 8K/1K line does. + // GLM's AgentX fixture lacks valid P90 metrics, so it cannot produce a tier read. const glm = page.models.find((m) => m.model === Model.GLM_5_2)!; const glmB300 = headlinePairOf(glm, 'b300-vs-b200')!; expect(glmB300.candidate.read.value).toBeNull(); - expect(glmB300.candidate.missingReason).toBe('no_8k1k_data'); + expect(glmB300.candidate.missingReason).toBe('no_scenario_data'); const communityPage = assembleOverviewPageData( overviewRowsFixture as unknown as Record, @@ -817,6 +885,6 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { const communityGlm = communityPage.models.find((m) => m.model === Model.GLM_5_2)!; const communityGlmB300 = headlinePairOf(communityGlm, 'b300-vs-b200')!; expect(communityGlmB300.candidate.read.value).toBeNull(); - expect(communityGlmB300.candidate.missingReason).toBe('no_8k1k_data'); + expect(communityGlmB300.candidate.missingReason).toBe('no_scenario_data'); }); }); diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 2ca2a3bf3..b315f7f4c 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -1,18 +1,25 @@ import { resolveFrameworkPartLabel } from '@semianalysisai/inferencex-constants'; +import { restrictAgenticPointsToE2eFrontier } from './agentic-frontier'; import type { BenchmarkRow } from './api'; +import { rowToAggDataEntry } from './benchmark-transform'; import { buildAvailabilityHwKey } from './chart-utils'; import { getGpuSpecs, getHardwareConfig } from './constants'; -import { DEFAULT_MODELS, getModelLabel, Precision, type Model } from './data-mappings'; +import { DEFAULT_MODELS, getModelLabel, Model, Precision } from './data-mappings'; import { frameworkFamily } from './framework-family'; -import { computeTcoFeed, type TcoTierBoundary } from './tco-feed'; +import { + computeTcoFeed, + computeTierReads, + type TcoTierBoundary, + type TcoTierPoint, +} from './tco-feed'; export const OVERVIEW_WORKLOAD = { isl: 8192, osl: 1024 } as const; export const OVERVIEW_TIERS = [30, 50, 75, 100] as const; export type OverviewTier = (typeof OVERVIEW_TIERS)[number]; export const OVERVIEW_PRIMARY_TIER = 50; export type OverviewEngineScope = 'all' | 'community'; -export type OverviewDecodeMode = 'speculative' | 'standard'; +export type OverviewScenario = 'single_turn_8k1k' | 'agentx'; export function resolveOverviewEngineScope( raw: string | string[] | undefined, @@ -72,7 +79,8 @@ export interface OverviewTierRead { * under-swept. */ export type OverviewMissingReason = | 'int4_bf16_only' - | 'no_8k1k_data' + | 'no_scenario_data' + | 'no_speculative_decode_result' | 'cannot_reach_at_tier' | 'no_exact_at_tier'; @@ -82,7 +90,6 @@ export interface OverviewPlatformResult { hardware: string; hardwareLabel: string; precision: string | null; - decodeMode: OverviewDecodeMode | null; read: OverviewTierRead; missingReason: OverviewMissingReason | null; /** $ per million output tokens at the retail-rental $/GPU/hr tier. */ @@ -95,6 +102,7 @@ export interface OverviewPlatformResult { export interface OverviewModelSummary { model: Model; modelLabel: string; + scenario: OverviewScenario; platforms: OverviewPlatformResult[]; } @@ -105,15 +113,7 @@ export interface OverviewPageData { engineScope: OverviewEngineScope; } -const OVERVIEW_SLICE_PRIORITY = [ - { decodeMode: 'speculative', precision: Precision.FP4 }, - { decodeMode: 'speculative', precision: Precision.FP8 }, - { decodeMode: 'standard', precision: Precision.FP4 }, - { decodeMode: 'standard', precision: Precision.FP8 }, -] as const satisfies readonly { - decodeMode: OverviewDecodeMode; - precision: string; -}[]; +const OVERVIEW_SLICE_PRIORITY = [Precision.FP4, Precision.FP8] as const; const OVERVIEW_PRECISIONS: readonly string[] = [Precision.FP4, Precision.FP8]; const OVERVIEW_HARDWARE_LABELS: Readonly> = { gb200: 'GB200', @@ -124,8 +124,11 @@ function overviewHardwareLabel(hardware: string, model: Model): string { return OVERVIEW_HARDWARE_LABELS[hardware] ?? getHardwareConfig(hardware, model).label; } -function decodeModeForSpecMethod(specMethod: string): OverviewDecodeMode { - return specMethod === 'none' || specMethod === '' ? 'standard' : 'speculative'; +const isSpeculativeDecode = (specMethod: string): boolean => + specMethod !== 'none' && specMethod !== ''; + +export function overviewScenarioForModel(model: Model): OverviewScenario { + return model === Model.Kimi_K3 || model === Model.GLM_5_2 ? 'agentx' : 'single_turn_8k1k'; } function overviewEngineRows( @@ -139,7 +142,10 @@ function overviewEngineRows( }); } -function overviewWorkloadRows(rows: readonly BenchmarkRow[]): BenchmarkRow[] { +function overviewScenarioRows(model: Model, rows: readonly BenchmarkRow[]): BenchmarkRow[] { + if (overviewScenarioForModel(model) === 'agentx') { + return rows.filter((row) => row.benchmark_type === 'agentic_traces'); + } return rows.filter( (row) => row.benchmark_type === 'single_turn' && @@ -148,13 +154,10 @@ function overviewWorkloadRows(rows: readonly BenchmarkRow[]): BenchmarkRow[] { ); } -/** Deliberately from raw rows, not retained winners: an unranked precision or - * engine still dates the dataset it was measured in. */ -export function overviewDatasetThroughDate( - rows: readonly BenchmarkRow[], - engineScope: OverviewEngineScope = 'community', -): string | null { - return overviewWorkloadRows(overviewEngineRows(rows, engineScope)).reduce( +/** Deliberately from raw scenario rows, not retained winners: an unranked + * precision or engine still dates the dataset it was measured in. */ +function latestOverviewDate(rows: readonly BenchmarkRow[]): string | null { + return rows.reduce( (latest, row) => (latest === null || row.date > latest ? row.date : latest), null, ); @@ -173,10 +176,15 @@ function overviewServingSeriesKey(row: BenchmarkRow): string { } /** Chart-equivalent serving series: topology variants are points on one curve. */ -function buildConfigs(model: Model, workloadRows: readonly BenchmarkRow[]): OverviewConfigResult[] { +function buildConfigs( + model: Model, + scenario: OverviewScenario, + scenarioRows: readonly BenchmarkRow[], +): OverviewConfigResult[] { const rowsByConfig = new Map(); - for (const row of workloadRows) { + for (const row of scenarioRows) { if (!OVERVIEW_PRECISIONS.includes(row.precision)) continue; + if (!isSpeculativeDecode(row.spec_method)) continue; const key = overviewServingSeriesKey(row); const configRows = rowsByConfig.get(key); if (configRows) configRows.push(row); @@ -190,7 +198,7 @@ function buildConfigs(model: Model, workloadRows: readonly BenchmarkRow[]): Over configRows[0].date, ); const latestRows = configRows.filter((row) => row.date === latestDate); - const config = buildConfigResult(model, latestRows[0].precision, key, latestRows); + const config = buildConfigResult(model, scenario, latestRows[0].precision, key, latestRows); if (config) configs.push(config); } return configs; @@ -254,10 +262,8 @@ function nonComparableAsMissing( } function configPriorityIndex(config: OverviewConfigResult): number { - return OVERVIEW_SLICE_PRIORITY.findIndex( - (priority) => - priority.precision === config.precision && - priority.decodeMode === decodeModeForSpecMethod(config.specMethod), + return OVERVIEW_SLICE_PRIORITY.indexOf( + config.precision as (typeof OVERVIEW_SLICE_PRIORITY)[number], ); } @@ -274,19 +280,19 @@ function selectPlatformRead( const exact = reads .filter( (read) => - read.config.precision === priority.precision && - decodeModeForSpecMethod(read.config.specMethod) === priority.decodeMode && + read.config.precision === priority && + isSpeculativeDecode(read.config.specMethod) && isInRangeTierRead(read), ) .toSorted(compareTierReads)[0]; if (exact) return exact; } - const fallback = reads.toSorted( + const bestMissingRead = reads.toSorted( (a, b) => configPriorityIndex(a.config) - configPriorityIndex(b.config) || compareTierReads(a, b), )[0]; - return fallback ? nonComparableAsMissing(fallback, tier) : nullTierRead(tier); + return bestMissingRead ? nonComparableAsMissing(bestMissingRead, tier) : nullTierRead(tier); } function missingReasonForPlatform( @@ -297,8 +303,12 @@ function missingReasonForPlatform( ): OverviewMissingReason | null { if (isInRangeTierRead(read)) return null; const hardwareRows = workloadRows.filter((row) => row.hardware === hardware); - if (hardwareRows.length === 0) return 'no_8k1k_data'; - const supportedRows = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); + if (hardwareRows.length === 0) return 'no_scenario_data'; + const speculativeRows = hardwareRows.filter((row) => isSpeculativeDecode(row.spec_method)); + if (speculativeRows.length === 0) return 'no_speculative_decode_result'; + const supportedRows = speculativeRows.filter((row) => + OVERVIEW_PRECISIONS.includes(row.precision), + ); if (supportedRows.length === 0) return 'int4_bf16_only'; // `cannot reach` is a claim about the whole platform, so it holds only when // EVERY qualified serving series tops out below the tier — one merely @@ -320,10 +330,11 @@ export function overviewCostPerMtok( function buildPlatformResults( model: Model, - workloadRows: readonly BenchmarkRow[], + scenario: OverviewScenario, + scenarioRows: readonly BenchmarkRow[], tier: OverviewTier, ): OverviewPlatformResult[] { - const configs = buildConfigs(model, workloadRows); + const configs = buildConfigs(model, scenario, scenarioRows); const readsForHardware = (hardware: string): OverviewTierRead[] => configs .filter((config) => config.hardware === hardware) @@ -335,10 +346,9 @@ function buildPlatformResults( hardware, hardwareLabel: overviewHardwareLabel(hardware, model), precision: read.config?.precision ?? null, - decodeMode: read.config === null ? null : decodeModeForSpecMethod(read.config.specMethod), read, missingReason: missingReasonForPlatform( - workloadRows, + scenarioRows, hardware, read, readsForHardware(hardware), @@ -357,37 +367,82 @@ function buildPlatformResults( })); } +function deployedGpuFactor(row: BenchmarkRow): number { + const totalGpus = row.num_prefill_gpu + row.num_decode_gpu; + return row.disagg && row.num_prefill_gpu > 0 && row.num_decode_gpu > 0 && totalGpus > 0 + ? row.num_decode_gpu / totalGpus + : 1; +} + +function topologyEvidence(row: BenchmarkRow): string | undefined { + return row.disagg && row.num_prefill_gpu > 0 && row.num_decode_gpu > 0 + ? `${row.num_prefill_gpu}P+${row.num_decode_gpu}D` + : undefined; +} + +interface OverviewAgenticTierPoint extends TcoTierPoint { + e2eLatency: number; + throughput: number; +} + +function buildAgenticTierReads(rows: readonly BenchmarkRow[]) { + const points = rows.flatMap((row): OverviewAgenticTierPoint[] => { + const entry = rowToAggDataEntry(row); + const factor = deployedGpuFactor(row); + const interactivity = entry.p90_intvty; + const e2eLatency = entry.p90_e2el; + const outputThroughput = entry.output_tput_per_gpu * factor; + if ( + !Number.isFinite(interactivity) || + interactivity <= 0 || + !Number.isFinite(e2eLatency) || + e2eLatency <= 0 || + !Number.isFinite(outputThroughput) || + outputThroughput <= 0 + ) { + return []; + } + return [ + { + interactivity, + e2eLatency, + throughput: outputThroughput, + outputThroughput, + date: row.date, + evidenceLabel: topologyEvidence(row), + }, + ]; + }); + return computeTierReads(restrictAgenticPointsToE2eFrontier(points), OVERVIEW_TIERS); +} + function buildConfigResult( model: Model, + scenario: OverviewScenario, precision: string, key: string, rows: BenchmarkRow[], ): OverviewConfigResult | null { - const feedRows = rows.map((row) => { - const outputTputPerGpu = row.metrics.output_tput_per_gpu; - const totalGpus = row.num_prefill_gpu + row.num_decode_gpu; - const comparableOutputTputPerGpu = - row.disagg && - row.num_prefill_gpu > 0 && - row.num_decode_gpu > 0 && - totalGpus > 0 && - Number.isFinite(outputTputPerGpu) - ? (outputTputPerGpu * row.num_decode_gpu) / totalGpus - : outputTputPerGpu; - const evidenceLabel = - row.disagg && row.num_prefill_gpu > 0 && row.num_decode_gpu > 0 - ? `${row.num_prefill_gpu}P+${row.num_decode_gpu}D` - : undefined; - return { - ...row, - metrics: { - ...row.metrics, - output_tput_per_gpu: comparableOutputTputPerGpu, - }, - evidence_label: evidenceLabel, - }; - }); - const feed = computeTcoFeed(feedRows, [OVERVIEW_WORKLOAD], OVERVIEW_TIERS); + const feed = + scenario === 'agentx' + ? buildAgenticTierReads(rows) + : computeTcoFeed( + rows.map((row) => { + const outputTputPerGpu = row.metrics.output_tput_per_gpu; + return { + ...row, + metrics: { + ...row.metrics, + output_tput_per_gpu: Number.isFinite(outputTputPerGpu) + ? outputTputPerGpu * deployedGpuFactor(row) + : outputTputPerGpu, + }, + evidence_label: topologyEvidence(row), + }; + }), + [OVERVIEW_WORKLOAD], + OVERVIEW_TIERS, + ); if (feed.length === 0) return null; const first = rows[0]; @@ -432,10 +487,13 @@ export function buildOverviewModelSummary( engineScope: OverviewEngineScope = 'community', ): OverviewModelSummary { const scopedRows = overviewEngineRows(rows, engineScope); + const scenario = overviewScenarioForModel(model); + const scenarioRows = overviewScenarioRows(model, scopedRows); return { model, modelLabel: getModelLabel(model), - platforms: buildPlatformResults(model, overviewWorkloadRows(scopedRows), tier), + scenario, + platforms: buildPlatformResults(model, scenario, scenarioRows, tier), }; } @@ -451,9 +509,10 @@ export function assembleOverviewPageData( models: perModel.map(({ model, rows }) => buildOverviewModelSummary(model, rows, tier, engineScope), ), - datasetThroughDate: overviewDatasetThroughDate( - perModel.flatMap(({ rows }) => rows), - engineScope, + datasetThroughDate: latestOverviewDate( + perModel.flatMap(({ model, rows }) => + overviewScenarioRows(model, overviewEngineRows(rows, engineScope)), + ), ), tier, engineScope, diff --git a/packages/app/src/lib/overview-links.test.ts b/packages/app/src/lib/overview-links.test.ts index 3711d1554..31950c81e 100644 --- a/packages/app/src/lib/overview-links.test.ts +++ b/packages/app/src/lib/overview-links.test.ts @@ -55,6 +55,7 @@ function summary(overrides: Partial = {}): OverviewModelSu return { model: Model.Qwen3_5, modelLabel: 'Qwen 3.5', + scenario: 'single_turn_8k1k', platforms: [], ...overrides, }; @@ -93,6 +94,18 @@ describe('buildOverviewDashboardHref', () => { expect(href).toContain('i_prec=fp8'); }); + it('opens AgentX evidence in the Agentic Traces dashboard scenario', () => { + const href = buildOverviewDashboardHref( + 'en', + summary({ model: Model.GLM_5_2, scenario: 'agentx' }), + config(), + ); + + expect(href).toContain('g_model=GLM-5.2'); + expect(href).toContain('i_seq=agentic-traces'); + expect(href).not.toContain('i_seq=8k%2F1k'); + }); + it('maps specMethod to the dashboard mtp/stp filter bucket, not the raw DB value', () => { expect(buildOverviewDashboardHref('en', summary(), config({ specMethod: 'eagle' }))).toContain( 'i_spec=mtp', @@ -115,6 +128,12 @@ describe('detailHref', () => { '/inference?g_model=Qwen-3.5-397B-A17B&i_seq=8k%2F1k&i_optimal=1', ); }); + + it('opens AgentX rows in the Agentic Traces dashboard scenario', () => { + expect(detailHref('en', summary({ model: Model.GLM_5_2, scenario: 'agentx' }))).toBe( + '/inference?g_model=GLM-5.2&i_seq=agentic-traces&i_optimal=1', + ); + }); }); describe('overviewHref', () => { diff --git a/packages/app/src/lib/overview-links.ts b/packages/app/src/lib/overview-links.ts index ab8db3a3c..24e444458 100644 --- a/packages/app/src/lib/overview-links.ts +++ b/packages/app/src/lib/overview-links.ts @@ -8,8 +8,9 @@ import { } from './overview-data'; import type { UrlStateParams } from './url-state'; -/** The single-turn 8K-in/1K-out workload every overview link filters to. */ -const OVERVIEW_WORKLOAD_SEQ = '8k/1k'; +function overviewSequence(model: OverviewModelSummary): '8k/1k' | 'agentic-traces' { + return model.scenario === 'agentx' ? 'agentic-traces' : '8k/1k'; +} /** The `/inference` route base for a locale — shared by every overview link. */ function inferenceRoute(locale: 'en' | 'zh'): string { @@ -57,7 +58,7 @@ export function buildOverviewDashboardHref( g_model: model.model, g_rundate: config.latestDate, g_runid: soleSourceRun(config)?.id, - i_seq: OVERVIEW_WORKLOAD_SEQ, + i_seq: overviewSequence(model), i_prec: config.precision, i_metric: 'y_outputTputPerGpu', i_gpus: config.hwKey, @@ -81,7 +82,7 @@ export function buildOverviewDashboardHref( export function detailHref(locale: 'en' | 'zh', model: OverviewModelSummary): string { const query = new URLSearchParams({ g_model: model.model, - i_seq: OVERVIEW_WORKLOAD_SEQ, + i_seq: overviewSequence(model), i_optimal: '1', }); return `${inferenceRoute(locale)}?${query}`; diff --git a/packages/app/src/lib/tco-feed.ts b/packages/app/src/lib/tco-feed.ts index 0003fdf0e..ced6be3d1 100644 --- a/packages/app/src/lib/tco-feed.ts +++ b/packages/app/src/lib/tco-feed.ts @@ -81,6 +81,15 @@ export interface TcoFeedRow { evidence_labels?: string[]; } +export interface TcoTierPoint { + interactivity: number; + outputThroughput: number; + date: string; + evidenceLabel?: string; +} + +export type TcoTierRead = Omit; + export const DEFAULT_TIERS: readonly number[] = [30, 50, 75, 100]; export const DEFAULT_WORKLOADS: readonly TcoFeedWorkload[] = [ { isl: 1024, osl: 1024 }, @@ -207,21 +216,14 @@ export function parseAlpha(raw: string | null): number | null { return value; } -interface FrontierPoint { - interactivity: number; - throughput: number; - date: string; - evidenceLabel?: string; -} - const round3 = (v: number): number => Math.round(v * 1000) / 1000; /** Two knots' dates as an ascending {from,to} evidence range. */ -function evidenceRange(a: FrontierPoint, b: FrontierPoint): { from: string; to: string } { +function evidenceRange(a: TcoTierPoint, b: TcoTierPoint): { from: string; to: string } { return a.date <= b.date ? { from: a.date, to: b.date } : { from: b.date, to: a.date }; } -function evidenceLabels(a: FrontierPoint, b: FrontierPoint): string[] | undefined { +function evidenceLabels(a: TcoTierPoint, b: TcoTierPoint): string[] | undefined { const labels = [...new Set([a.evidenceLabel, b.evidenceLabel].filter(Boolean))] as string[]; return labels.length === 0 ? undefined : labels; } @@ -231,7 +233,7 @@ function evidenceLabels(a: FrontierPoint, b: FrontierPoint): string[] | undefine * pair xs[lo] ≤ tier ≤ xs[lo+1], collapsing to a single knot when `tier` lands * on an endpoint. `frontier` is sorted ascending by interactivity. */ -function bracketKnots(frontier: FrontierPoint[], tier: number): [FrontierPoint, FrontierPoint] { +function bracketKnots(frontier: TcoTierPoint[], tier: number): [TcoTierPoint, TcoTierPoint] { const last = frontier.length - 1; if (tier <= frontier[0].interactivity) return [frontier[0], frontier[0]]; if (tier >= frontier[last].interactivity) return [frontier[last], frontier[last]]; @@ -243,6 +245,72 @@ function bracketKnots(frontier: FrontierPoint[], tier: number): [FrontierPoint, return [frontier[lo], frontier[lo + 1]]; } +/** Read fixed service tiers from an output-throughput serving frontier. */ +export function computeTierReads( + points: readonly TcoTierPoint[], + tiers: readonly number[], +): TcoTierRead[] { + const frontier = paretoFrontUpperLeft( + [...points], + (point) => point.interactivity, + (point) => point.outputThroughput, + ).toSorted((a, b) => a.interactivity - b.interactivity); + if (frontier.length === 0) return []; + + const xs = frontier.map((point) => point.interactivity); + const ys = frontier.map((point) => point.outputThroughput); + const slopes = monotoneSlopes(xs, ys); + const minIv = xs[0]; + const maxIv = xs.at(-1)!; + const yLo = Math.min(...ys); + const yHi = Math.max(...ys); + let latest = frontier[0].date; + let oldest = frontier[0].date; + for (const point of frontier) { + if (point.date > latest) latest = point.date; + if (point.date < oldest) oldest = point.date; + } + + return tiers.map((tier) => { + let value: number; + let boundary: TcoTierBoundary; + let isInterpolated = false; + let evidenceDate: { from: string; to: string } | null; + let tierEvidenceLabels: string[] | undefined; + if (tier > maxIv) { + value = 0; + boundary = 'unreachable'; + evidenceDate = null; + } else if (tier < minIv) { + value = ys[0]; + boundary = 'clamped_low'; + evidenceDate = evidenceRange(frontier[0], frontier[0]); + tierEvidenceLabels = evidenceLabels(frontier[0], frontier[0]); + } else { + const [lo, hi] = bracketKnots(frontier, tier); + const raw = hermiteInterpolate(xs, ys, slopes, tier); + value = Math.max(yLo, Math.min(yHi, raw)); + boundary = 'interpolated'; + isInterpolated = lo.interactivity !== hi.interactivity; + evidenceDate = evidenceRange(lo, hi); + tierEvidenceLabels = evidenceLabels(lo, hi); + } + return { + tier, + output_tput_per_gpu: round3(value), + boundary, + is_interpolated: isInterpolated, + frontier_points: frontier.length, + frontier_min_interactivity: round3(minIv), + frontier_max_interactivity: round3(maxIv), + latest_date: latest, + oldest_frontier_date: oldest, + evidence_date: evidenceDate, + ...(tierEvidenceLabels === undefined ? {} : { evidence_labels: tierEvidenceLabels }), + }; + }); +} + /** * Compute the feed: for each (workload, hardware), build the * (interactivity, output tok/s/GPU) Pareto frontier across every config — @@ -256,7 +324,7 @@ export function computeTcoFeed( const out: TcoFeedRow[] = []; for (const workload of workloads) { - const byHardware = new Map(); + const byHardware = new Map(); for (const row of rows) { // Rows predating the benchmark_type column are fixed-seq by definition // (agentic rows are newer and carry isl/osl = null, which the workload @@ -281,7 +349,7 @@ export function computeTcoFeed( if (!Number.isFinite(otput) || otput <= 0) continue; const point = { interactivity, - throughput: otput, + outputThroughput: otput, date: row.date, evidenceLabel: row.evidence_label, }; @@ -293,68 +361,11 @@ export function computeTcoFeed( const workloadKey = `${workload.isl}x${workload.osl}`; const hardwareKeys = [...byHardware.keys()].toSorted((a, b) => a.localeCompare(b)); for (const hardware of hardwareKeys) { - const frontier = paretoFrontUpperLeft( - byHardware.get(hardware)!, - (p) => p.interactivity, - (p) => p.throughput, - ).toSorted((a, b) => a.interactivity - b.interactivity); - if (frontier.length === 0) continue; - - const xs = frontier.map((p) => p.interactivity); - const ys = frontier.map((p) => p.throughput); - const slopes = monotoneSlopes(xs, ys); - const minIv = xs[0]; - const maxIv = xs.at(-1)!; - // Clamp bounds against spline overshoot, mirroring interpolateForGPU. - const yLo = Math.min(...ys); - const yHi = Math.max(...ys); - // 'YYYY-MM-DD' compares chronologically as a string. - let latest = frontier[0].date; - let oldest = frontier[0].date; - for (const p of frontier) { - if (p.date > latest) latest = p.date; - if (p.date < oldest) oldest = p.date; - } - - for (const tier of tiers) { - let value: number; - let boundary: TcoTierBoundary; - let isInterpolated = false; - let evidenceDate: { from: string; to: string } | null; - let tierEvidenceLabels: string[] | undefined; - if (tier > maxIv) { - value = 0; - boundary = 'unreachable'; - evidenceDate = null; - } else if (tier < minIv) { - value = ys[0]; - boundary = 'clamped_low'; - // The clamped read is the min-interactivity knot's throughput. - evidenceDate = evidenceRange(frontier[0], frontier[0]); - tierEvidenceLabels = evidenceLabels(frontier[0], frontier[0]); - } else { - const [lo, hi] = bracketKnots(frontier, tier); - const raw = hermiteInterpolate(xs, ys, slopes, tier); - value = Math.max(yLo, Math.min(yHi, raw)); - boundary = 'interpolated'; - isInterpolated = lo.interactivity !== hi.interactivity; - evidenceDate = evidenceRange(lo, hi); - tierEvidenceLabels = evidenceLabels(lo, hi); - } + for (const read of computeTierReads(byHardware.get(hardware)!, tiers)) { out.push({ hardware, workload: workloadKey, - tier, - output_tput_per_gpu: round3(value), - boundary, - is_interpolated: isInterpolated, - frontier_points: frontier.length, - frontier_min_interactivity: round3(minIv), - frontier_max_interactivity: round3(maxIv), - latest_date: latest, - oldest_frontier_date: oldest, - evidence_date: evidenceDate, - ...(tierEvidenceLabels === undefined ? {} : { evidence_labels: tierEvidenceLabels }), + ...read, }); } } From 77c734c458e460facf4d5bb3f79bfb3e03b1851e Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 29 Jul 2026 15:32:18 -0700 Subject: [PATCH 2/4] fix(overview): address review feedback --- packages/app/cypress/e2e/overview.cy.ts | 17 ++-- packages/app/cypress/e2e/zh-pages.cy.ts | 11 +-- .../src/app/zh/(dashboard)/overview/page.tsx | 2 +- .../src/components/landing/landing-page.tsx | 94 ++++++++++++------- .../overview/overview-scorecard.tsx | 45 ++++----- packages/app/src/lib/overview-data.test.ts | 22 +++++ packages/app/src/lib/overview-data.ts | 3 +- 7 files changed, 119 insertions(+), 75 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 2b825679e..6556906bc 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -409,7 +409,9 @@ describe('Overview page', () => { .click(); cy.location('pathname').should('eq', '/zh/overview'); cy.location('search').should('eq', '?tier=100'); - cy.contains('GPU 租赁成本 / 每百万输出 token · @100 tok/s/用户 · ↓ 越低越好').should('exist'); + cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @100 tok/s/用户 · ↓ 越低越好').should( + 'exist', + ); }); it('uses the same cell semantics on mobile and fits both 390px and 320px widths', () => { @@ -603,12 +605,10 @@ describe('Overview page', () => { cy.contains('一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。').should( 'exist', ); - cy.contains('按各模型标注的场景，基于各平台最佳观测服务包络线计算每百万输出 token 成本').should( - 'exist', - ); + cy.contains('基于最佳观测平台服务包络线计算的各活跃模型每百万输出 token 成本').should('exist'); cy.get('[data-testid="overview-scope"]').should( 'have.text', - 'GPU 租赁成本 / 每百万输出 token · @50 tok/s/用户 · ↓ 越低越好', + 'GPU 租赁成本 / 每百万输出 token · 8K→1K · @50 tok/s/用户 · ↓ 越低越好', ); cy.contains('成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。').should('exist'); cy.contains('分离式结果的分母同时计入预填充与解码 GPU。').should('exist'); @@ -636,14 +636,15 @@ describe('Overview page', () => { platform('b200').find('[data-testid="overview-cost-delta"]').should('not.exist'); }); desktopModel('GLM-5.2').within(() => { - cy.get('[data-testid="overview-model-scenario"]').should('have.text', 'AgentX'); + cy.get('[data-testid="overview-model-scenario"]').should('not.exist'); cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); }); - cy.get('body').should('not.contain.text', '标准解码'); cy.contains('∞ = 无可比结果').should('exist'); cy.visit('/zh/overview?tier=100'); - cy.contains('GPU 租赁成本 / 每百万输出 token · @100 tok/s/用户 · ↓ 越低越好').should('exist'); + cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @100 tok/s/用户 · ↓ 越低越好').should( + 'exist', + ); cy.get('[data-testid="overview-tier-switcher"]').within(() => { cy.contains('a', '50').should('have.attr', 'href', '/zh/overview'); }); diff --git a/packages/app/cypress/e2e/zh-pages.cy.ts b/packages/app/cypress/e2e/zh-pages.cy.ts index 70c0a6a85..77fc8cfc5 100644 --- a/packages/app/cypress/e2e/zh-pages.cy.ts +++ b/packages/app/cypress/e2e/zh-pages.cy.ts @@ -5,17 +5,12 @@ describe('Chinese (/zh) pages', () => { }); it('renders the Chinese landing content', () => { - cy.contains('h2', '探索 InferenceX').should('exist'); + cy.contains('h2', '完整仪表板').should('exist'); cy.contains('快速对比').should('exist'); }); - it('links to the Chinese overview and full dashboard', () => { - cy.get('[data-testid="landing-overview-link"]') - .should('have.attr', 'href', '/zh/overview') - .and('have.text', '总览'); - cy.get('[data-testid="landing-full-dashboard-link"]') - .should('have.attr', 'href', '/zh/inference') - .and('have.text', '完整仪表板'); + it('links into the Chinese dashboard tree', () => { + cy.get('a[href="/zh/inference"]').should('exist'); }); it('sets hreflang alternates to the English homepage', () => { diff --git a/packages/app/src/app/zh/(dashboard)/overview/page.tsx b/packages/app/src/app/zh/(dashboard)/overview/page.tsx index e3aaf44a6..5ddeb6fd3 100644 --- a/packages/app/src/app/zh/(dashboard)/overview/page.tsx +++ b/packages/app/src/app/zh/(dashboard)/overview/page.tsx @@ -10,7 +10,7 @@ import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; const DESCRIPTION = - '按各活跃模型标注的场景，对比 MI355X、B200、B300、GB200 与 GB300 的每百万输出 token 成本。'; + '在固定单轮 8K 输入 / 1K 输出负载下，基于最佳观测平台服务包络线对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的每百万输出 token 成本；优先采用推测解码与 FP4。'; export const metadata: Metadata = { title: '推理成本总览', diff --git a/packages/app/src/components/landing/landing-page.tsx b/packages/app/src/components/landing/landing-page.tsx index 02f10f294..cfae3ac1b 100644 --- a/packages/app/src/components/landing/landing-page.tsx +++ b/packages/app/src/components/landing/landing-page.tsx @@ -40,13 +40,12 @@ const STRINGS = { 'Jump straight into the most popular GPU inference benchmark comparisons, curated and ready to explore.', }, zh: { - exploreInferenceX: '探索 InferenceX', - exploreInferenceXLead: - '先查看活跃模型与关键平台的精简成本总览，或打开完整仪表板，探索所有模型、GPU、框架与指标。', fullDashboard: '完整仪表板', - platformCoverage: + fullDashboardP1: + '覆盖所有模型、GPU、框架与指标。完全可配置的推理基准测试图表，支持日期范围、并发扫描与原始数据导出。', + fullDashboardP2: '跨 DeepSeekv4 Pro、Qwen、Kimi、GLM、MiniMax、gpt-oss、Llama 等模型，对比 NVIDIA GB300 NVL72、GB200 NVL72、B300、B200、H200、H100、AMD MI355X、MI325X、MI300X，以及即将上线的 VR200 NVL72、AMD MI455X UALoE72、TPUv7 Ironwood 等硬件。', - overview: '总览', + openDashboard: '打开仪表板', reproTitle: '每一条结果都通过公开的 GitHub Actions 自动化流程透明产生', reproP1: '仪表板上的每个数据点都由公开的 GitHub Actions 工作流运行产生。配置方案（recipe）保存在公开仓库中，运行在真实目标硬件上执行，完整日志与产物公开可查。点击图表上的任意数据点即可跳转到生成它的那次运行。一切都可复现、可审计、开源。', @@ -80,37 +79,60 @@ export function LandingPage({ locale = 'en' }: { locale?: Locale } = {}) { {/* Split: exploration entry points vs presets */}
- {/* Primary product entry points */} - -
- -

{t.exploreInferenceX}

-
-

{t.exploreInferenceXLead}

-

{t.platformCoverage}

-
- - {t.overview} - - - {t.fullDashboard} - -
- + {locale === 'zh' ? ( + +
+ +

{STRINGS.zh.fullDashboard}

+
+

{STRINGS.zh.fullDashboardP1}

+

{STRINGS.zh.fullDashboardP2}

+
+ + {STRINGS.zh.openDashboard} + + +
+ + ) : ( + +
+ +

{STRINGS.en.exploreInferenceX}

+
+

+ {STRINGS.en.exploreInferenceXLead} +

+

{STRINGS.en.platformCoverage}

+
+ + {STRINGS.en.overview} + + + {STRINGS.en.fullDashboard} + +
+ + )} {/* Reproducibility callout */} diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index 732563109..965dc6bb1 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -72,7 +72,8 @@ export const OVERVIEW_STRINGS = { zh: { title: '推理成本总览', purpose: '一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。', - scope: (tier: number) => `GPU 租赁成本 / 每百万输出 token · @${tier} tok/s/用户 · ↓ 越低越好`, + scope: (tier: number) => + `GPU 租赁成本 / 每百万输出 token · 8K→1K · @${tier} tok/s/用户 · ↓ 越低越好`, tierNavLabel: '服务档位', tierUnit: 'tok/s/用户', engineScopeNavLabel: '引擎范围', @@ -81,31 +82,29 @@ export const OVERVIEW_STRINGS = { community: '开源社区引擎（vLLM/SGLang）', }, snapshot: (through: string) => `数据库快照截至 ${through}`, - caption: '按各模型标注的场景，基于各平台最佳观测服务包络线计算每百万输出 token 成本。', - modelHeader: '模型 · 场景', - scenarioLabels: { - single_turn_8k1k: '单轮 · 8K→1K', - agentx: 'AgentX', - }, + caption: + '基于最佳观测平台服务包络线计算的各活跃模型每百万输出 token 成本；优先采用推测解码与 FP4。', + modelHeader: '模型', detailsHeader: '详情', detailLink: '查看详情', detailAria: (modelLabel: string) => `查看详情：${modelLabel}`, rawDashboardAria: (evidenceDate: string, modelLabel: string, stack: string) => `打开 ${evidenceDate} 原始数据仪表板：${modelLabel} · ${stack}`, + standardDecode: '标准解码', estimatedTooltip: (topologies: readonly string[]) => topologies.length === 0 ? '根据已验证的基准运行结果估算。' : `根据已验证的 ${topologies.join(' 与 ')} 运行结果估算。`, estimatedAria: (value: string, explanation: string) => `约 ${value}。${explanation}`, + noWorkloadResults: '暂无 8K/1K 结果', infinityLegend: '∞ = 无可比结果', missingReasons: (tier: number): Record => ({ int4_bf16_only: '仅 INT4/BF16', - no_scenario_data: '该场景暂无数据', - no_speculative_decode_result: '暂无推测解码结果', + no_8k1k_data: '无 8K/1K 数据', cannot_reach_at_tier: `无法达到 @${tier}`, no_exact_at_tier: `无精确 @${tier} 结果`, }), - methodologyNote: '仅展示推测解码。优先顺序：FP4 → FP8。', + methodologyNote: '优先顺序：推测解码 FP4 → 推测解码 FP8 → 标准解码 FP4 → 标准解码 FP8。', costNote: '成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。所有百分比均相对 B200。', costDeltaAria: (pct: string, cheaper: boolean) => `比 B200 ${cheaper ? '便宜' : '昂贵'} ${pct}`, costDeltaEvenAria: '与 B200 成本基本持平', @@ -113,7 +112,8 @@ export const OVERVIEW_STRINGS = { normalizationNote: '分离式结果的分母同时计入预填充与解码 GPU。', interpolationNote: '各档位数值采用最佳观测平台服务包络线；≈ 表示根据已验证运行结果估算。不会外推。', - comparabilityNote: '每行均在该模型标注的场景内比较各平台；日期、引擎、精度与推测方法可能不同。', + comparabilityNote: + '方向性平台对比：各单元格取该平台最佳观测包络线，日期、引擎、精度与解码方式可能不同。', }, } as const; @@ -162,7 +162,8 @@ function formatEvidenceDate( function missingReasonCopy(platform: OverviewPlatformResult, strings: OverviewStrings): string { const reason = platform.missingReason; - return reason === null ? '' : strings.missingReasons(platform.read.tier)[reason]; + if (reason === null) return ''; + return strings.missingReasons(platform.read.tier)[reason] ?? strings.infinityLegend; } const RAW_SOURCE_LINK_CLASS = @@ -381,16 +382,18 @@ function PlatformCell(props: { ); } -function ModelName({ model, strings }: { model: OverviewModelSummary; strings: OverviewStrings }) { +function ModelName({ model, locale }: { model: OverviewModelSummary; locale: OverviewLocale }) { return (

{model.modelLabel}

-

- {strings.scenarioLabels[model.scenario]} -

+ {locale === 'en' ? ( +

+ {OVERVIEW_STRINGS.en.scenarioLabels[model.scenario]} +

+ ) : null}
); } @@ -445,7 +448,7 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S className="border-b border-border/50 align-top last:border-b-0" > - + {model.platforms.map((platform) => ( - +
{model.platforms.map((platform) => (
{ }); }); + it('reports scenario-level missing coverage when AgentX rows lack usable P90 metrics', () => { + const summary = buildOverviewModelSummary(Model.GLM_5_2, [ + row({ + model: 'glm5.2', + hardware: 'b200', + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + precision: Precision.FP4, + spec_method: 'mtp', + metrics: { + output_tput_per_gpu: 1200, + }, + }), + ]); + + expect(summary.platforms.find(({ hardware }) => hardware === 'b200')).toMatchObject({ + read: { value: null }, + missingReason: 'no_scenario_data', + }); + }); + it('returns all five platforms with coverage gaps for an empty model', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, []); diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index b315f7f4c..6415bb890 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -310,10 +310,11 @@ function missingReasonForPlatform( OVERVIEW_PRECISIONS.includes(row.precision), ); if (supportedRows.length === 0) return 'int4_bf16_only'; + if (bucketReads.length === 0) return 'no_scenario_data'; // `cannot reach` is a claim about the whole platform, so it holds only when // EVERY qualified serving series tops out below the tier — one merely // under-swept stack downgrades the gap to a missing exact read. - return bucketReads.length > 0 && bucketReads.every((r) => r.boundary === 'unreachable') + return bucketReads.every((r) => r.boundary === 'unreachable') ? 'cannot_reach_at_tier' : 'no_exact_at_tier'; } From 18734e3b86d3c1d0eb91315c04c192794af730b4 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 29 Jul 2026 16:07:18 -0700 Subject: [PATCH 3/4] fix(overview): restore Chinese feature parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the Chinese landing CTAs, scenario-aware overview copy, and specific missing-result reasons after removing the temporary English-only override. 中文：恢复中文落地页入口、场景感知总览文案与具体缺失结果原因，确保删除临时仅英文覆盖规则后中英文功能保持一致。 --- packages/app/cypress/e2e/overview.cy.ts | 23 +++-- packages/app/cypress/e2e/zh-pages.cy.ts | 11 ++- .../src/app/zh/(dashboard)/overview/page.tsx | 2 +- .../src/components/landing/landing-page.tsx | 94 +++++++------------ .../overview/overview-scorecard.tsx | 45 +++++---- 5 files changed, 80 insertions(+), 95 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 6556906bc..d6bb2a6ca 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -409,9 +409,7 @@ describe('Overview page', () => { .click(); cy.location('pathname').should('eq', '/zh/overview'); cy.location('search').should('eq', '?tier=100'); - cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @100 tok/s/用户 · ↓ 越低越好').should( - 'exist', - ); + cy.contains('GPU 租赁成本 / 每百万输出 token · @100 tok/s/用户 · ↓ 越低越好').should('exist'); }); it('uses the same cell semantics on mobile and fits both 390px and 320px widths', () => { @@ -605,10 +603,12 @@ describe('Overview page', () => { cy.contains('一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。').should( 'exist', ); - cy.contains('基于最佳观测平台服务包络线计算的各活跃模型每百万输出 token 成本').should('exist'); + cy.contains('按各模型标注的场景，基于各平台最佳观测服务包络线计算每百万输出 token 成本').should( + 'exist', + ); cy.get('[data-testid="overview-scope"]').should( 'have.text', - 'GPU 租赁成本 / 每百万输出 token · 8K→1K · @50 tok/s/用户 · ↓ 越低越好', + 'GPU 租赁成本 / 每百万输出 token · @50 tok/s/用户 · ↓ 越低越好', ); cy.contains('成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。').should('exist'); cy.contains('分离式结果的分母同时计入预填充与解码 GPU。').should('exist'); @@ -633,18 +633,23 @@ describe('Overview page', () => { .should('not.match', /100 档由.+领先/); desktopModel('Kimi-K2.5').within(() => { cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); + platform('b200') + .find('[data-testid="overview-pair-missing"]') + .should('have.attr', 'title', '暂无推测解码结果'); platform('b200').find('[data-testid="overview-cost-delta"]').should('not.exist'); }); desktopModel('GLM-5.2').within(() => { - cy.get('[data-testid="overview-model-scenario"]').should('not.exist'); + cy.get('[data-testid="overview-model-scenario"]').should('have.text', 'AgentX'); cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); + platform('b300') + .find('[data-testid="overview-pair-missing"]') + .should('have.attr', 'title', '该场景暂无数据'); }); + cy.get('body').should('not.contain.text', '标准解码'); cy.contains('∞ = 无可比结果').should('exist'); cy.visit('/zh/overview?tier=100'); - cy.contains('GPU 租赁成本 / 每百万输出 token · 8K→1K · @100 tok/s/用户 · ↓ 越低越好').should( - 'exist', - ); + cy.contains('GPU 租赁成本 / 每百万输出 token · @100 tok/s/用户 · ↓ 越低越好').should('exist'); cy.get('[data-testid="overview-tier-switcher"]').within(() => { cy.contains('a', '50').should('have.attr', 'href', '/zh/overview'); }); diff --git a/packages/app/cypress/e2e/zh-pages.cy.ts b/packages/app/cypress/e2e/zh-pages.cy.ts index 77fc8cfc5..70c0a6a85 100644 --- a/packages/app/cypress/e2e/zh-pages.cy.ts +++ b/packages/app/cypress/e2e/zh-pages.cy.ts @@ -5,12 +5,17 @@ describe('Chinese (/zh) pages', () => { }); it('renders the Chinese landing content', () => { - cy.contains('h2', '完整仪表板').should('exist'); + cy.contains('h2', '探索 InferenceX').should('exist'); cy.contains('快速对比').should('exist'); }); - it('links into the Chinese dashboard tree', () => { - cy.get('a[href="/zh/inference"]').should('exist'); + it('links to the Chinese overview and full dashboard', () => { + cy.get('[data-testid="landing-overview-link"]') + .should('have.attr', 'href', '/zh/overview') + .and('have.text', '总览'); + cy.get('[data-testid="landing-full-dashboard-link"]') + .should('have.attr', 'href', '/zh/inference') + .and('have.text', '完整仪表板'); }); it('sets hreflang alternates to the English homepage', () => { diff --git a/packages/app/src/app/zh/(dashboard)/overview/page.tsx b/packages/app/src/app/zh/(dashboard)/overview/page.tsx index 5ddeb6fd3..e3aaf44a6 100644 --- a/packages/app/src/app/zh/(dashboard)/overview/page.tsx +++ b/packages/app/src/app/zh/(dashboard)/overview/page.tsx @@ -10,7 +10,7 @@ import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; const DESCRIPTION = - '在固定单轮 8K 输入 / 1K 输出负载下，基于最佳观测平台服务包络线对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的每百万输出 token 成本；优先采用推测解码与 FP4。'; + '按各活跃模型标注的场景，对比 MI355X、B200、B300、GB200 与 GB300 的每百万输出 token 成本。'; export const metadata: Metadata = { title: '推理成本总览', diff --git a/packages/app/src/components/landing/landing-page.tsx b/packages/app/src/components/landing/landing-page.tsx index cfae3ac1b..02f10f294 100644 --- a/packages/app/src/components/landing/landing-page.tsx +++ b/packages/app/src/components/landing/landing-page.tsx @@ -40,12 +40,13 @@ const STRINGS = { 'Jump straight into the most popular GPU inference benchmark comparisons, curated and ready to explore.', }, zh: { + exploreInferenceX: '探索 InferenceX', + exploreInferenceXLead: + '先查看活跃模型与关键平台的精简成本总览，或打开完整仪表板，探索所有模型、GPU、框架与指标。', fullDashboard: '完整仪表板', - fullDashboardP1: - '覆盖所有模型、GPU、框架与指标。完全可配置的推理基准测试图表，支持日期范围、并发扫描与原始数据导出。', - fullDashboardP2: + platformCoverage: '跨 DeepSeekv4 Pro、Qwen、Kimi、GLM、MiniMax、gpt-oss、Llama 等模型，对比 NVIDIA GB300 NVL72、GB200 NVL72、B300、B200、H200、H100、AMD MI355X、MI325X、MI300X，以及即将上线的 VR200 NVL72、AMD MI455X UALoE72、TPUv7 Ironwood 等硬件。', - openDashboard: '打开仪表板', + overview: '总览', reproTitle: '每一条结果都通过公开的 GitHub Actions 自动化流程透明产生', reproP1: '仪表板上的每个数据点都由公开的 GitHub Actions 工作流运行产生。配置方案（recipe）保存在公开仓库中，运行在真实目标硬件上执行，完整日志与产物公开可查。点击图表上的任意数据点即可跳转到生成它的那次运行。一切都可复现、可审计、开源。', @@ -79,60 +80,37 @@ export function LandingPage({ locale = 'en' }: { locale?: Locale } = {}) { {/* Split: exploration entry points vs presets */}
- {locale === 'zh' ? ( - -
- -

{STRINGS.zh.fullDashboard}

-
-

{STRINGS.zh.fullDashboardP1}

-

{STRINGS.zh.fullDashboardP2}

-
- - {STRINGS.zh.openDashboard} - - -
- - ) : ( - -
- -

{STRINGS.en.exploreInferenceX}

-
-

- {STRINGS.en.exploreInferenceXLead} -

-

{STRINGS.en.platformCoverage}

-
- - {STRINGS.en.overview} - - - {STRINGS.en.fullDashboard} - -
- - )} + {/* Primary product entry points */} + +
+ +

{t.exploreInferenceX}

+
+

{t.exploreInferenceXLead}

+

{t.platformCoverage}

+
+ + {t.overview} + + + {t.fullDashboard} + +
+ {/* Reproducibility callout */} diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index 965dc6bb1..732563109 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -72,8 +72,7 @@ export const OVERVIEW_STRINGS = { zh: { title: '推理成本总览', purpose: '一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。', - scope: (tier: number) => - `GPU 租赁成本 / 每百万输出 token · 8K→1K · @${tier} tok/s/用户 · ↓ 越低越好`, + scope: (tier: number) => `GPU 租赁成本 / 每百万输出 token · @${tier} tok/s/用户 · ↓ 越低越好`, tierNavLabel: '服务档位', tierUnit: 'tok/s/用户', engineScopeNavLabel: '引擎范围', @@ -82,29 +81,31 @@ export const OVERVIEW_STRINGS = { community: '开源社区引擎（vLLM/SGLang）', }, snapshot: (through: string) => `数据库快照截至 ${through}`, - caption: - '基于最佳观测平台服务包络线计算的各活跃模型每百万输出 token 成本；优先采用推测解码与 FP4。', - modelHeader: '模型', + caption: '按各模型标注的场景，基于各平台最佳观测服务包络线计算每百万输出 token 成本。', + modelHeader: '模型 · 场景', + scenarioLabels: { + single_turn_8k1k: '单轮 · 8K→1K', + agentx: 'AgentX', + }, detailsHeader: '详情', detailLink: '查看详情', detailAria: (modelLabel: string) => `查看详情：${modelLabel}`, rawDashboardAria: (evidenceDate: string, modelLabel: string, stack: string) => `打开 ${evidenceDate} 原始数据仪表板：${modelLabel} · ${stack}`, - standardDecode: '标准解码', estimatedTooltip: (topologies: readonly string[]) => topologies.length === 0 ? '根据已验证的基准运行结果估算。' : `根据已验证的 ${topologies.join(' 与 ')} 运行结果估算。`, estimatedAria: (value: string, explanation: string) => `约 ${value}。${explanation}`, - noWorkloadResults: '暂无 8K/1K 结果', infinityLegend: '∞ = 无可比结果', missingReasons: (tier: number): Record => ({ int4_bf16_only: '仅 INT4/BF16', - no_8k1k_data: '无 8K/1K 数据', + no_scenario_data: '该场景暂无数据', + no_speculative_decode_result: '暂无推测解码结果', cannot_reach_at_tier: `无法达到 @${tier}`, no_exact_at_tier: `无精确 @${tier} 结果`, }), - methodologyNote: '优先顺序：推测解码 FP4 → 推测解码 FP8 → 标准解码 FP4 → 标准解码 FP8。', + methodologyNote: '仅展示推测解码。优先顺序：FP4 → FP8。', costNote: '成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。所有百分比均相对 B200。', costDeltaAria: (pct: string, cheaper: boolean) => `比 B200 ${cheaper ? '便宜' : '昂贵'} ${pct}`, costDeltaEvenAria: '与 B200 成本基本持平', @@ -112,8 +113,7 @@ export const OVERVIEW_STRINGS = { normalizationNote: '分离式结果的分母同时计入预填充与解码 GPU。', interpolationNote: '各档位数值采用最佳观测平台服务包络线；≈ 表示根据已验证运行结果估算。不会外推。', - comparabilityNote: - '方向性平台对比：各单元格取该平台最佳观测包络线，日期、引擎、精度与解码方式可能不同。', + comparabilityNote: '每行均在该模型标注的场景内比较各平台；日期、引擎、精度与推测方法可能不同。', }, } as const; @@ -162,8 +162,7 @@ function formatEvidenceDate( function missingReasonCopy(platform: OverviewPlatformResult, strings: OverviewStrings): string { const reason = platform.missingReason; - if (reason === null) return ''; - return strings.missingReasons(platform.read.tier)[reason] ?? strings.infinityLegend; + return reason === null ? '' : strings.missingReasons(platform.read.tier)[reason]; } const RAW_SOURCE_LINK_CLASS = @@ -382,18 +381,16 @@ function PlatformCell(props: { ); } -function ModelName({ model, locale }: { model: OverviewModelSummary; locale: OverviewLocale }) { +function ModelName({ model, strings }: { model: OverviewModelSummary; strings: OverviewStrings }) { return (

{model.modelLabel}

- {locale === 'en' ? ( -

- {OVERVIEW_STRINGS.en.scenarioLabels[model.scenario]} -

- ) : null} +

+ {strings.scenarioLabels[model.scenario]} +

); } @@ -448,7 +445,7 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S className="border-b border-border/50 align-top last:border-b-0" > - + {model.platforms.map((platform) => ( - +
{model.platforms.map((platform) => (
Date: Wed, 29 Jul 2026 16:50:56 -0700 Subject: [PATCH 4/4] fix(overview): restore standard-decode fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prefer speculative FP4/FP8 per platform, then fall back to labelled standard FP4/FP8 results. Select AgentX only when a model has no single-turn 8K/1K rows.\n\n中文：各平台优先选择推测解码 FP4/FP8；缺失时回退到明确标注的标准解码 FP4/FP8。仅当模型没有单轮 8K/1K 数据时选择 AgentX。 --- packages/app/cypress/e2e/overview.cy.ts | 30 ++--- .../overview/overview-scorecard.tsx | 20 ++-- packages/app/src/lib/overview-data.test.ts | 107 +++++++++++++++--- packages/app/src/lib/overview-data.ts | 58 ++++++---- 4 files changed, 158 insertions(+), 57 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index d6bb2a6ca..ee8391201 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -127,26 +127,27 @@ describe('Overview page', () => { }); }); - it('shows only speculative-decode reads, with FP8 and method labels', () => { + it('prefers speculative decode and falls back to labelled standard-decode reads', () => { cy.viewport(1280, 900); cy.visit('/overview'); desktopModel('Kimi-K2.5').within(() => { + cy.get('[data-testid="overview-pair-value"]').should('have.length', 3); cy.get('[data-testid="overview-pair-missing"]') - .should('have.length', 5) + .should('have.length', 2) .then(($missing) => { expect([...$missing].map((element) => element.getAttribute('title'))).to.deep.equal([ - 'no speculative-decode result', - 'no speculative-decode result', - 'no speculative-decode result', 'no data for this scenario', 'no data for this scenario', ]); }); cy.get('[data-testid="overview-pair-missing"]') .children('[aria-hidden="true"]') - .should('have.length', 5) - .and('have.text', '∞∞∞∞∞'); + .should('have.length', 2) + .and('have.text', '∞∞'); + platform('b200').should('contain.text', 'Standard decode'); + platform('mi355x').should('contain.text', 'Standard decode'); + platform('b300').should('contain.text', 'Standard decode'); }); desktopModel('DeepSeek-V4-Pro').within(() => { @@ -164,7 +165,9 @@ describe('Overview page', () => { cy.get('[data-testid="overview-pair-missing"]').should('not.exist'); }); }); - cy.get('body').should('not.contain.text', 'Standard decode'); + cy.contains( + 'Priority: speculative FP4 → speculative FP8 → standard FP4 → standard FP8.', + ).should('exist'); cy.get('body').should('not.contain.text', 'P90'); }); @@ -632,10 +635,9 @@ describe('Overview page', () => { .invoke('text') .should('not.match', /100 档由.+领先/); desktopModel('Kimi-K2.5').within(() => { - cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); - platform('b200') - .find('[data-testid="overview-pair-missing"]') - .should('have.attr', 'title', '暂无推测解码结果'); + cy.get('[data-testid="overview-pair-value"]').should('have.length', 3); + cy.get('[data-testid="overview-pair-missing"]').should('have.length', 2); + platform('b200').should('contain.text', '标准解码'); platform('b200').find('[data-testid="overview-cost-delta"]').should('not.exist'); }); desktopModel('GLM-5.2').within(() => { @@ -645,7 +647,9 @@ describe('Overview page', () => { .find('[data-testid="overview-pair-missing"]') .should('have.attr', 'title', '该场景暂无数据'); }); - cy.get('body').should('not.contain.text', '标准解码'); + cy.contains('优先顺序：推测解码 FP4 → 推测解码 FP8 → 标准解码 FP4 → 标准解码 FP8。').should( + 'exist', + ); cy.contains('∞ = 无可比结果').should('exist'); cy.visit('/zh/overview?tier=100'); diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index 732563109..1c5a3c1c7 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -51,11 +51,11 @@ export const OVERVIEW_STRINGS = { missingReasons: (tier: number): Record => ({ int4_bf16_only: 'INT4/BF16 only', no_scenario_data: 'no data for this scenario', - no_speculative_decode_result: 'no speculative-decode result', cannot_reach_at_tier: `cannot reach @${tier}`, no_exact_at_tier: `no exact @${tier} result`, }), - methodologyNote: 'Speculative decode only. Priority: FP4 → FP8.', + standardDecodeLabel: 'Standard decode', + methodologyNote: 'Priority: speculative FP4 → speculative FP8 → standard FP4 → standard FP8.', costNote: 'Cost = 3-yr rental $/GPU/hr ÷ output tok/s per deployed GPU. All percentages compare against B200.', costDeltaAria: (pct: string, cheaper: boolean) => @@ -101,11 +101,11 @@ export const OVERVIEW_STRINGS = { missingReasons: (tier: number): Record => ({ int4_bf16_only: '仅 INT4/BF16', no_scenario_data: '该场景暂无数据', - no_speculative_decode_result: '暂无推测解码结果', cannot_reach_at_tier: `无法达到 @${tier}`, no_exact_at_tier: `无精确 @${tier} 结果`, }), - methodologyNote: '仅展示推测解码。优先顺序：FP4 → FP8。', + standardDecodeLabel: '标准解码', + methodologyNote: '优先顺序：推测解码 FP4 → 推测解码 FP8 → 标准解码 FP4 → 标准解码 FP8。', costNote: '成本 = 3 年期租赁 $/GPU/小时 ÷ 每张已部署 GPU 的输出 tok/s。所有百分比均相对 B200。', costDeltaAria: (pct: string, cheaper: boolean) => `比 B200 ${cheaper ? '便宜' : '昂贵'} ${pct}`, costDeltaEvenAria: '与 B200 成本基本持平', @@ -266,10 +266,16 @@ function CellValue({ return ; } const precisionLabel = config?.precision.toUpperCase() ?? member.precision?.toUpperCase() ?? null; + const specLabel = + config === null + ? null + : config.specMethod === 'none' || config.specMethod === '' + ? strings.standardDecodeLabel + : config.specLabel; const stackBadge = - config === null || precisionLabel === null + config === null || precisionLabel === null || specLabel === null ? null - : [config.frameworkLabel, precisionLabel, config.specLabel] + : [config.frameworkLabel, precisionLabel, specLabel] .filter((part): part is string => part !== null) .join(' · '); const stack = @@ -279,7 +285,7 @@ function CellValue({ member.hardwareLabel, config.frameworkLabel, config.precision.toUpperCase(), - config.specLabel, + specLabel, ].join(' · '); const evidenceDateLabel = evidenceDate === null ? '' : formatEvidenceDate(formatters, evidenceDate); diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 498940da3..02aabf5af 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -95,6 +95,37 @@ describe('overview engine scope and scenario selection', () => { expect(overviewScenarioForModel(Model.Qwen3_5)).toBe('single_turn_8k1k'); }); + it('prefers single-turn 8K/1K rows and otherwise falls back to AgentX', () => { + const singleTurn = frontier([1200, 1000, 800, 600], { + model: 'glm5.2', + hardware: 'b200', + }); + const agentx = [ + row({ + model: 'glm5.2', + hardware: 'b200', + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + metrics: { + p90_itl: 1 / 50, + p90_ttlt: 25, + output_tput_per_gpu: 850, + }, + }), + ]; + + expect(buildOverviewModelSummary(Model.GLM_5_2, [...agentx, ...singleTurn]).scenario).toBe( + 'single_turn_8k1k', + ); + expect( + buildOverviewModelSummary( + Model.Qwen3_5, + agentx.map((entry) => ({ ...entry, model: 'qwen3.5' })), + ).scenario, + ).toBe('agentx'); + }); + it('resolves valid engine scopes and defaults invalid values to community', () => { expect(resolveOverviewEngineScope('community')).toBe('community'); expect(resolveOverviewEngineScope('all')).toBe('all'); @@ -420,7 +451,7 @@ describe('overview engine scope and scenario selection', () => { expect(headlinePairOf(summary, 'gb300-vs-b200')?.candidate.read.value).toBeNull(); }); - it('uses speculative FP4 then speculative FP8 without standard-decode fallback', () => { + it('uses speculative FP4, speculative FP8, standard FP4, then standard FP8', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([900, 700, 500, 300], { hardware: 'b200', precision: Precision.FP4 }), ...frontier([1400, 1200, 1000, 800], { @@ -465,9 +496,9 @@ describe('overview engine scope and scenario selection', () => { ).toEqual([ { hardware: 'b200', precision: Precision.FP4, value: 700 }, { hardware: 'mi355x', precision: Precision.FP8, value: 900 }, - { hardware: 'b300', precision: null, value: null }, - { hardware: 'gb200', precision: null, value: null }, - { hardware: 'gb300', precision: null, value: null }, + { hardware: 'b300', precision: Precision.FP4, value: 1100 }, + { hardware: 'gb200', precision: Precision.FP8, value: 1000 }, + { hardware: 'gb300', precision: Precision.FP4, value: 900 }, ]); }); }); @@ -623,20 +654,53 @@ describe('overview platform selection', () => { ); }); - it('shows standard-only data as missing and still flags unsupported precision coverage', () => { + it('falls back to standard decode and still flags unsupported precision coverage', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), - row({ hardware: 'mi355x', precision: Precision.FP8, spec_method: 'none' }), + ...frontier([1100, 900, 700, 500], { + hardware: 'mi355x', + precision: Precision.FP8, + spec_method: 'none', + }), row({ hardware: 'b300', precision: Precision.INT4 }), ]); expect(headlinePairOf(summary, 'mi355x-vs-b200')?.candidate).toMatchObject({ - missingReason: 'no_speculative_decode_result', - read: { value: null }, + missingReason: null, + precision: Precision.FP8, + read: { value: 900, config: { specMethod: 'none' } }, }); expect(headlinePairOf(summary, 'b300-vs-b200')?.candidate.missingReason).toBe('int4_bf16_only'); }); + it('falls back to standard decode for AgentX when no speculative result exists', () => { + const agentxRows = [40, 50, 60].map((interactivity, index) => + row({ + model: 'glm5.2', + hardware: 'b300', + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + precision: Precision.FP4, + spec_method: 'none', + conc: index + 1, + metrics: { + p90_itl: 1 / interactivity, + p90_ttlt: 30 - index * 5, + output_tput_per_gpu: 1000 - index * 100, + }, + }), + ); + + const summary = buildOverviewModelSummary(Model.GLM_5_2, agentxRows); + expect(summary.scenario).toBe('agentx'); + expect(summary.platforms.find(({ hardware }) => hardware === 'b300')).toMatchObject({ + missingReason: null, + precision: Precision.FP4, + read: { value: 900, config: { specMethod: 'none' } }, + }); + }); + it('reads AgentX at the chart-default P90 contract without exposing P90 as the scenario', () => { const summary = buildOverviewModelSummary(Model.GLM_5_2, [ row({ @@ -880,18 +944,27 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { expect(qwenB300.baseline.precision).toBe(Precision.FP4); expect(qwenB300.baseline.read.value).toBeCloseTo(733.594); - // Kimi: standard-only rows remain visible as spec-decode coverage gaps. + // Kimi: standard-only rows remain visible as explicitly labelled fallbacks. const kimi = page.models.find((m) => m.model === Model.Kimi_K2_5)!; const kimiMi = headlinePairOf(kimi, 'mi355x-vs-b200')!; - expect(kimiMi.candidate.precision).toBeNull(); - expect(kimiMi.candidate.read.value).toBeNull(); - expect(kimiMi.candidate.missingReason).toBe('no_speculative_decode_result'); - expect(kimiMi.baseline.precision).toBeNull(); - expect(kimiMi.baseline.read.value).toBeNull(); + expect(kimiMi.candidate.precision).toBe(Precision.FP4); + expect(kimiMi.candidate.read).toMatchObject({ + value: 1000, + config: { specMethod: 'none' }, + }); + expect(kimiMi.candidate.missingReason).toBeNull(); + expect(kimiMi.baseline.precision).toBe(Precision.FP4); + expect(kimiMi.baseline.read).toMatchObject({ + value: 800, + config: { specMethod: 'none' }, + }); const kimiB300 = headlinePairOf(kimi, 'b300-vs-b200')!; - expect(kimiB300.candidate.precision).toBeNull(); - expect(kimiB300.candidate.read.value).toBeNull(); - expect(kimiB300.candidate.missingReason).toBe('no_speculative_decode_result'); + expect(kimiB300.candidate.precision).toBe(Precision.FP8); + expect(kimiB300.candidate.read).toMatchObject({ + value: 900, + config: { specMethod: 'none' }, + }); + expect(kimiB300.candidate.missingReason).toBeNull(); // GLM's AgentX fixture lacks valid P90 metrics, so it cannot produce a tier read. const glm = page.models.find((m) => m.model === Model.GLM_5_2)!; diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 6415bb890..33cc0e409 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -80,7 +80,6 @@ export interface OverviewTierRead { export type OverviewMissingReason = | 'int4_bf16_only' | 'no_scenario_data' - | 'no_speculative_decode_result' | 'cannot_reach_at_tier' | 'no_exact_at_tier'; @@ -113,7 +112,12 @@ export interface OverviewPageData { engineScope: OverviewEngineScope; } -const OVERVIEW_SLICE_PRIORITY = [Precision.FP4, Precision.FP8] as const; +const OVERVIEW_SLICE_PRIORITY = [ + { speculative: true, precision: Precision.FP4 }, + { speculative: true, precision: Precision.FP8 }, + { speculative: false, precision: Precision.FP4 }, + { speculative: false, precision: Precision.FP8 }, +] as const; const OVERVIEW_PRECISIONS: readonly string[] = [Precision.FP4, Precision.FP8]; const OVERVIEW_HARDWARE_LABELS: Readonly> = { gb200: 'GB200', @@ -127,7 +131,21 @@ function overviewHardwareLabel(hardware: string, model: Model): string { const isSpeculativeDecode = (specMethod: string): boolean => specMethod !== 'none' && specMethod !== ''; -export function overviewScenarioForModel(model: Model): OverviewScenario { +export function overviewScenarioForModel( + model: Model, + rows: readonly BenchmarkRow[] = [], +): OverviewScenario { + if ( + rows.some( + (row) => + row.benchmark_type === 'single_turn' && + row.isl === OVERVIEW_WORKLOAD.isl && + row.osl === OVERVIEW_WORKLOAD.osl, + ) + ) { + return 'single_turn_8k1k'; + } + if (rows.some((row) => row.benchmark_type === 'agentic_traces')) return 'agentx'; return model === Model.Kimi_K3 || model === Model.GLM_5_2 ? 'agentx' : 'single_turn_8k1k'; } @@ -142,8 +160,11 @@ function overviewEngineRows( }); } -function overviewScenarioRows(model: Model, rows: readonly BenchmarkRow[]): BenchmarkRow[] { - if (overviewScenarioForModel(model) === 'agentx') { +function overviewScenarioRows( + scenario: OverviewScenario, + rows: readonly BenchmarkRow[], +): BenchmarkRow[] { + if (scenario === 'agentx') { return rows.filter((row) => row.benchmark_type === 'agentic_traces'); } return rows.filter( @@ -184,7 +205,6 @@ function buildConfigs( const rowsByConfig = new Map(); for (const row of scenarioRows) { if (!OVERVIEW_PRECISIONS.includes(row.precision)) continue; - if (!isSpeculativeDecode(row.spec_method)) continue; const key = overviewServingSeriesKey(row); const configRows = rowsByConfig.get(key); if (configRows) configRows.push(row); @@ -262,8 +282,9 @@ function nonComparableAsMissing( } function configPriorityIndex(config: OverviewConfigResult): number { - return OVERVIEW_SLICE_PRIORITY.indexOf( - config.precision as (typeof OVERVIEW_SLICE_PRIORITY)[number], + return OVERVIEW_SLICE_PRIORITY.findIndex( + ({ speculative, precision }) => + speculative === isSpeculativeDecode(config.specMethod) && precision === config.precision, ); } @@ -280,8 +301,8 @@ function selectPlatformRead( const exact = reads .filter( (read) => - read.config.precision === priority && - isSpeculativeDecode(read.config.specMethod) && + read.config.precision === priority.precision && + isSpeculativeDecode(read.config.specMethod) === priority.speculative && isInRangeTierRead(read), ) .toSorted(compareTierReads)[0]; @@ -304,11 +325,7 @@ function missingReasonForPlatform( if (isInRangeTierRead(read)) return null; const hardwareRows = workloadRows.filter((row) => row.hardware === hardware); if (hardwareRows.length === 0) return 'no_scenario_data'; - const speculativeRows = hardwareRows.filter((row) => isSpeculativeDecode(row.spec_method)); - if (speculativeRows.length === 0) return 'no_speculative_decode_result'; - const supportedRows = speculativeRows.filter((row) => - OVERVIEW_PRECISIONS.includes(row.precision), - ); + const supportedRows = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); if (supportedRows.length === 0) return 'int4_bf16_only'; if (bucketReads.length === 0) return 'no_scenario_data'; // `cannot reach` is a claim about the whole platform, so it holds only when @@ -488,8 +505,8 @@ export function buildOverviewModelSummary( engineScope: OverviewEngineScope = 'community', ): OverviewModelSummary { const scopedRows = overviewEngineRows(rows, engineScope); - const scenario = overviewScenarioForModel(model); - const scenarioRows = overviewScenarioRows(model, scopedRows); + const scenario = overviewScenarioForModel(model, rows); + const scenarioRows = overviewScenarioRows(scenario, scopedRows); return { model, modelLabel: getModelLabel(model), @@ -511,9 +528,10 @@ export function assembleOverviewPageData( buildOverviewModelSummary(model, rows, tier, engineScope), ), datasetThroughDate: latestOverviewDate( - perModel.flatMap(({ model, rows }) => - overviewScenarioRows(model, overviewEngineRows(rows, engineScope)), - ), + perModel.flatMap(({ model, rows }) => { + const scenario = overviewScenarioForModel(model, rows); + return overviewScenarioRows(scenario, overviewEngineRows(rows, engineScope)); + }), ), tier, engineScope,