diff --git a/packages/app/cypress/component/scatter-graph.cy.tsx b/packages/app/cypress/component/scatter-graph.cy.tsx index aade1334..692f6a77 100644 --- a/packages/app/cypress/component/scatter-graph.cy.tsx +++ b/packages/app/cypress/component/scatter-graph.cy.tsx @@ -558,13 +558,19 @@ describe('ScatterGraph', () => { runUrl, }; const exclusion = buildExclusion([ - { suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] }, + { + suffix: null, + stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], + scope: 'hardware', + }, ]); const namespacedExclusion = { familyOf: (key: string) => exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), groupOf: (key: string) => exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), + scopesOf: (key: string) => + exclusion.scopesOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), }; const blockedToggle = cy.stub().as('blockedComparisonToggle').returns(null); @@ -649,13 +655,19 @@ describe('ScatterGraph', () => { runUrl, }; const exclusion = buildExclusion([ - { suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] }, + { + suffix: null, + stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], + scope: 'hardware', + }, ]); const namespacedExclusion = { familyOf: (key: string) => exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), groupOf: (key: string) => exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), + scopesOf: (key: string) => + exclusion.scopesOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), }; mountWithProviders( @@ -845,7 +857,7 @@ describe('ChartDisplay engine comparison guard', () => { cy.get('@setLocalOfficialOverride').should('not.have.been.called'); }); - it('keeps cross-engine AgentX STP rows out of table mode', () => { + it('keeps same-hardware cross-engine AgentX STP rows out of table mode', () => { const chartDefinition = createMockChartDefinition({ chartType: 'interactivity' }); const sglangRow = createMockInferenceData({ hwKey: 'b200_sglang', @@ -854,13 +866,17 @@ describe('ChartDisplay engine comparison guard', () => { precision: Precision.FP4, }); const vllmRow = createMockInferenceData({ - hwKey: 'h100_vllm', + hwKey: 'b200_vllm', hw: 'Official vLLM', model: Model.DeepSeek_V4_Pro, precision: Precision.FP4, }); const exclusion = buildExclusion([ - { suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] }, + { + suffix: null, + stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], + scope: 'hardware', + }, ]); const resolveSelection = (proposed: Set, prev = new Set()) => resolveExclusionGroups(proposed, prev, exclusion, 'keep-sticky'); @@ -879,7 +895,7 @@ describe('ChartDisplay engine comparison guard', () => { selectedSequence: Sequence.AgenticTraces, selectedXAxisMode: 'interactivity', activeHwTypes: new Set(['b200_sglang']), - hwTypesWithData: new Set(['b200_sglang', 'h100_vllm']), + hwTypesWithData: new Set(['b200_sglang', 'b200_vllm']), resolveComparisonSelection: resolveSelection, }, globalFilters: { @@ -911,13 +927,19 @@ describe('ChartDisplay engine comparison guard', () => { run_url: runUrl, }); const exclusion = buildExclusion([ - { suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] }, + { + suffix: null, + stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], + scope: 'hardware', + }, ]); const namespacedExclusion = { familyOf: (key: string) => exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), groupOf: (key: string) => exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), + scopesOf: (key: string) => + exclusion.scopesOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), }; const resolveSelection = (proposed: Set, prev = new Set()) => resolveExclusionGroups(proposed, prev, namespacedExclusion, 'keep-sticky'); @@ -1014,13 +1036,19 @@ describe('ChartDisplay engine comparison guard', () => { it('commits a new table overlay scope and preserves an explicit empty selection', () => { const chartDefinition = createMockChartDefinition({ chartType: 'interactivity' }); const exclusion = buildExclusion([ - { suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] }, + { + suffix: null, + stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], + scope: 'hardware', + }, ]); const namespacedExclusion = { familyOf: (key: string) => exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), groupOf: (key: string) => exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), + scopesOf: (key: string) => + exclusion.scopesOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), }; const resolveSelection = (proposed: Set, prev = new Set()) => resolveExclusionGroups(proposed, prev, namespacedExclusion, 'keep-sticky'); diff --git a/packages/app/src/components/engine-comparison-conflict-toast.test.ts b/packages/app/src/components/engine-comparison-conflict-toast.test.ts new file mode 100644 index 00000000..a0423b58 --- /dev/null +++ b/packages/app/src/components/engine-comparison-conflict-toast.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; + +import { describeEngineComparisonConflict } from './engine-comparison-conflict-toast'; + +describe('describeEngineComparisonConflict', () => { + it('describes partial removal without assuming hardware scope', () => { + const message = describeEngineComparisonConflict( + { + kind: 'resolved', + kept: ['vllm'], + dropped: [], + partial: ['sglang'], + }, + 'en', + ); + + expect(message).toContain( + 'Disabled conflicting SGLang configs while compatible SGLang configs remain shown', + ); + expect(message).not.toContain('SKU'); + expect(message).not.toContain('Kept SGLang'); + expect(message).not.toContain('removed SGLang'); + }); + + it('preserves the whole-family resolution message', () => { + expect( + describeEngineComparisonConflict( + { + kind: 'resolved', + kept: ['sglang'], + dropped: ['vllm'], + partial: [], + }, + 'en', + ), + ).toContain('Kept SGLang and removed vLLM configs'); + }); +}); diff --git a/packages/app/src/components/engine-comparison-conflict-toast.tsx b/packages/app/src/components/engine-comparison-conflict-toast.tsx index 1fd8d651..89bfabb3 100644 --- a/packages/app/src/components/engine-comparison-conflict-toast.tsx +++ b/packages/app/src/components/engine-comparison-conflict-toast.tsx @@ -17,7 +17,7 @@ import { BottomToast } from '@/components/ui/bottom-toast'; */ export type EngineComparisonConflictDetail = | { kind: 'blocked'; attempted: string; existing: string | null } - | { kind: 'resolved'; kept: string[]; dropped: string[] }; + | { kind: 'resolved'; kept: string[]; dropped: string[]; partial: string[] }; function familyLabel(family: string): string { return FRAMEWORK_LABELS[family] ?? family; @@ -37,7 +37,10 @@ function joinListZh(parts: string[]): string { return `${parts.slice(0, -1).join('、')}和 ${parts.at(-1)}`; } -function describe(detail: EngineComparisonConflictDetail, locale: Locale): string { +export function describeEngineComparisonConflict( + detail: EngineComparisonConflictDetail, + locale: Locale, +): string { if (locale === 'zh') return describeZh(detail); if (detail.kind === 'blocked') { const attempted = familyLabel(detail.attempted); @@ -49,6 +52,11 @@ function describe(detail: EngineComparisonConflictDetail, locale: Locale): strin } const kept = [...detail.kept].toSorted().map(familyLabel); const dropped = [...detail.dropped].toSorted().map(familyLabel); + const partial = [...detail.partial].toSorted().map(familyLabel); + if (partial.length > 0) { + const removed = dropped.length > 0 ? ` Removed ${joinList(dropped)} configs.` : ''; + return `Only compatible engine configurations can be shown together in this view.${removed} Disabled conflicting ${joinList(partial)} configs while compatible ${joinList(partial)} configs remain shown.`; + } if (kept.length > 0) { return `Only compatible engine families can be shown together in this view. Kept ${joinList(kept)} and removed ${joinList(dropped)} configs.`; } @@ -69,6 +77,11 @@ function describeZh(detail: EngineComparisonConflictDetail): string { } const kept = [...detail.kept].toSorted().map(familyLabel); const dropped = [...detail.dropped].toSorted().map(familyLabel); + const partial = [...detail.partial].toSorted().map(familyLabel); + if (partial.length > 0) { + const removed = dropped.length > 0 ? ` 已移除 ${joinListZh(dropped)} 配置。` : ''; + return `此视图只能同时显示相互兼容的引擎配置。${removed}已禁用冲突的 ${joinListZh(partial)} 配置，同时保留兼容的 ${joinListZh(partial)} 配置。`; + } if (kept.length > 0) { return `此视图只能同时显示相互兼容的引擎系列。已保留 ${joinListZh(kept)}，并移除 ${joinListZh(dropped)} 配置。`; } @@ -101,6 +114,7 @@ export function EngineComparisonConflictToast({ detail, onDismiss }: Props) { existing: detail.kind === 'blocked' ? detail.existing : null, kept: detail.kind === 'resolved' ? detail.kept : null, dropped: detail.kind === 'resolved' ? detail.dropped : null, + partial: detail.kind === 'resolved' ? detail.partial : null, }); }, [detail]); @@ -112,7 +126,7 @@ export function EngineComparisonConflictToast({ detail, onDismiss }: Props) { testId="engine-comparison-conflict-toast" icon={} title={TITLES[locale]} - description={describe(detail, locale)} + description={describeEngineComparisonConflict(detail, locale)} onDismiss={onDismiss} /> ); diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 5933bca5..b8af14ed 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -201,11 +201,12 @@ export function InferenceProvider({ if (!selectedGpuResolution) return; setSelectedGpuState(selectedGpuResolution.selection); setUrlParam('i_gpus', selectedGpuResolution.selection.join(',')); - if (selectedGpuResolution.dropped.length > 0) { + if (selectedGpuResolution.dropped.length > 0 || selectedGpuResolution.partial.length > 0) { setEngineConflict({ kind: 'resolved', kept: selectedGpuResolution.kept, dropped: selectedGpuResolution.dropped, + partial: selectedGpuResolution.partial, }); } }, [selectedGpuResolution, setUrlParam]); @@ -801,12 +802,16 @@ export function InferenceProvider({ exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), groupOf: (key: string) => exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), + scopesOf: (key: string) => + exclusion.scopesOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), } : null, [exclusion], ); const activeHwTypesRef = useRef(activeHwTypes); activeHwTypesRef.current = activeHwTypes; + const preferredHwTypesRef = useRef(activeHwTypes); + if (activeHwTypes.size > 1) preferredHwTypesRef.current = activeHwTypes; const exclusionRef = useRef(comparisonExclusion); exclusionRef.current = comparisonExclusion; const exclusionPolicyRef = useRef(exclusionPolicy); @@ -827,7 +832,7 @@ export function InferenceProvider({ (prev: Set, item: string, allItems: Set): Set | null => { const currentExclusion = exclusionRef.current; const toggleUniverse = currentExclusion - ? effectiveLegendItems(allItems, prev, currentExclusion) + ? effectiveLegendItems(allItems, prev, currentExclusion, preferredHwTypesRef.current) : allItems; if (currentExclusion) { const decision = resolveExclusionToggle( @@ -845,9 +850,14 @@ export function InferenceProvider({ }); return null; } - if (decision.kind === 'silent-resolve') return decision.result; + if (decision.kind === 'silent-resolve') { + if (decision.result.size > 1) preferredHwTypesRef.current = decision.result; + return decision.result; + } } - return computeToggle(prev, item, toggleUniverse); + const result = computeToggle(prev, item, toggleUniverse); + if (result.size > 1) preferredHwTypesRef.current = result; + return result; }, [], ); diff --git a/packages/app/src/components/inference/utils/comparison-exclusion.test.ts b/packages/app/src/components/inference/utils/comparison-exclusion.test.ts index a29f1613..0e6bf31d 100644 --- a/packages/app/src/components/inference/utils/comparison-exclusion.test.ts +++ b/packages/app/src/components/inference/utils/comparison-exclusion.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; +import { resolveExclusionToggle } from '@/lib/exclusion'; import { Model, Sequence } from '@/lib/data-mappings'; import { comparisonExclusion } from './comparison-exclusion'; @@ -12,6 +13,87 @@ describe('comparisonExclusion', () => { expect(exclusion?.familyOf('b200_sglang')).toBe('sglang'); }); + it.each([ + { + name: 'blocks Agentic STP engines on the same SKU', + sequence: Sequence.AgenticTraces, + active: 'b200_sglang', + candidate: 'b200_vllm', + expected: 'block', + }, + { + name: 'allows Agentic STP engines on different SKUs', + sequence: Sequence.AgenticTraces, + active: 'b200_sglang', + candidate: 'mi355x_vllm', + expected: 'fallthrough', + }, + { + name: 'blocks Agentic MTP added to cross-engine STP on the same SKU', + sequence: Sequence.AgenticTraces, + active: 'b200_sglang', + candidate: 'b200_vllm_mtp', + expected: 'block', + }, + { + name: 'blocks Agentic STP added to cross-engine MTP on the same SKU', + sequence: Sequence.AgenticTraces, + active: 'b200_vllm_mtp', + candidate: 'b200_sglang', + expected: 'block', + }, + { + name: 'allows Agentic STP and MTP engines on different SKUs', + sequence: Sequence.AgenticTraces, + active: 'b200_sglang', + candidate: 'mi355x_vllm_mtp', + expected: 'fallthrough', + }, + { + name: 'allows Agentic STP and MTP from the same engine', + sequence: Sequence.AgenticTraces, + active: 'b200_vllm', + candidate: 'b200_vllm_mtp', + expected: 'fallthrough', + }, + { + name: 'blocks Agentic cross-engine MTP globally', + sequence: Sequence.AgenticTraces, + active: 'b200_sglang_mtp', + candidate: 'mi355x_vllm_mtp', + expected: 'block', + }, + { + name: 'allows fixed-sequence STP engines on the same SKU', + sequence: Sequence.OneK_OneK, + active: 'b200_sglang', + candidate: 'b200_vllm', + expected: 'fallthrough', + }, + { + name: 'blocks fixed-sequence cross-engine MTP globally', + sequence: Sequence.OneK_OneK, + active: 'b200_sglang_mtp', + candidate: 'mi355x_vllm_mtp', + expected: 'block', + }, + ] as const)('$name', ({ sequence, active, candidate, expected }) => { + const exclusion = comparisonExclusion(Model.DeepSeek_V4_Pro, sequence, false)!; + const decision = resolveExclusionToggle( + new Set([active]), + candidate, + new Set([active, candidate]), + exclusion, + 'keep-sticky', + ); + + expect(decision.kind).toBe(expected); + }); + + it('does not create a guard outside configured models and scenarios', () => { + expect(comparisonExclusion(Model.Llama3_3_70B, Sequence.OneK_OneK, false)).toBeNull(); + }); + it('disables the engine-family guard for unofficial previews', () => { expect(comparisonExclusion(Model.DeepSeek_V4_Pro, Sequence.AgenticTraces, true)).toBeNull(); }); diff --git a/packages/app/src/lib/data-mappings.ts b/packages/app/src/lib/data-mappings.ts index 2b8fbde5..7c20a744 100644 --- a/packages/app/src/lib/data-mappings.ts +++ b/packages/app/src/lib/data-mappings.ts @@ -67,15 +67,17 @@ const MTP_ENGINE_EXCLUSION: ExclusionSpec[] = [ ]; /** - * AgentX STP exclusion: unsuffixed standard-token configs from different engine - * families can't be active together. Fixed-sequence STP comparisons remain - * available; this rule is attached only to the Agentic Traces sequence. + * AgentX STP exclusion: unsuffixed standard-token configs for the same hardware + * SKU can't mix engine families. Different hardware may use different engines + * on one graph. Fixed-sequence STP comparisons remain available; this rule is + * attached only to the Agentic Traces sequence. */ const AGENTIC_STP_ENGINE_EXCLUSION: ExclusionSpec[] = [ { suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], groupAliases: { atom: 'sglang' }, + scope: 'hardware', }, ]; diff --git a/packages/app/src/lib/exclusion.test.ts b/packages/app/src/lib/exclusion.test.ts index fe71221f..62cfb64a 100644 --- a/packages/app/src/lib/exclusion.test.ts +++ b/packages/app/src/lib/exclusion.test.ts @@ -28,6 +28,7 @@ const STP_SPEC: ExclusionSpec[] = [ suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'], groupAliases: { atom: 'sglang' }, + scope: 'hardware', }, ]; const agenticEx = buildExclusion([...MTP_SPEC, ...STP_SPEC]); @@ -36,6 +37,8 @@ const namespacedAgenticEx = { agenticEx.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), groupOf: (key: string) => agenticEx.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), + scopesOf: (key: string) => + agenticEx.scopesOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key), }; describe('buildExclusion — familyOf', () => { @@ -95,6 +98,18 @@ describe('exclusionResolutionFamilies', () => { expect(exclusionResolutionFamilies(proposed, result, ex)).toEqual({ kept: ['atom', 'sglang'], dropped: ['vllm'], + partial: [], + }); + }); + + it('reports families retained only on other hardware scopes as partial', () => { + const proposed = new Set(['b200_vllm', 'b200_sglang', 'mi355x_sglang']); + const result = new Set(['b200_vllm', 'mi355x_sglang']); + + expect(exclusionResolutionFamilies(proposed, result, agenticEx)).toEqual({ + kept: ['vllm'], + dropped: [], + partial: ['sglang'], }); }); }); @@ -120,6 +135,14 @@ describe('AgentX STP engine exclusion', () => { }); }); + it('allows different engine families on different hardware SKUs', () => { + const prev = new Set(['b200_sglang']); + const all = new Set(['b200_sglang', 'mi355x_vllm']); + expect(resolveExclusionToggle(prev, 'mi355x_vllm', all, agenticEx, 'keep-sticky')).toEqual({ + kind: 'fallthrough', + }); + }); + it('allows STP and MTP configs from the same engine family', () => { const prev = new Set(['b300_vllm']); const all = new Set(['b300_vllm', 'b300_vllm_mtp']); @@ -128,6 +151,16 @@ describe('AgentX STP engine exclusion', () => { }); }); + it('blocks cross-engine STP and MTP configs on the same hardware SKU', () => { + const prev = new Set(['b300_sglang']); + const all = new Set(['b300_sglang', 'b300_vllm_mtp']); + expect(resolveExclusionToggle(prev, 'b300_vllm_mtp', all, agenticEx, 'keep-sticky')).toEqual({ + kind: 'block', + attempted: 'vllm', + existing: 'sglang', + }); + }); + it('keeps the active engine during automatic AgentX selection resolution', () => { const proposed = new Set(['b300_sglang', 'b300_vllm', 'b300_vllm_mtp']); const resolved = resolveExclusionGroups( @@ -141,6 +174,36 @@ describe('AgentX STP engine exclusion', () => { expect(resolved.droppedGroups).toEqual(['sglang']); }); + it('uses hardware STP state to choose the compatible global MTP engine', () => { + const proposed = new Set(['b200_vllm', 'b200_vllm_mtp', 'mi355x_sglang_mtp']); + const resolved = resolveExclusionGroups( + proposed, + new Set(['b200_vllm']), + agenticEx, + 'keep-sticky', + ); + + expect(resolved.result).toEqual(new Set(['b200_vllm', 'b200_vllm_mtp'])); + expect(resolved.keptGroup).toBe('vllm'); + expect(resolved.droppedGroups).toEqual(['sglang']); + }); + + it.each([ + ['SGLang MTP first', ['b200_sglang', 'mi355x_vllm', 'b200_sglang_mtp', 'mi355x_vllm_mtp']], + ['vLLM MTP first', ['b200_sglang', 'mi355x_vllm', 'mi355x_vllm_mtp', 'b200_sglang_mtp']], + ])('breaks multiple correlated MTP ties alphabetically with %s', (_label, keys) => { + const resolved = resolveExclusionGroups( + new Set(keys), + new Set(['b200_sglang', 'mi355x_vllm']), + agenticEx, + 'keep-sticky', + ); + + expect(resolved.result).toEqual(new Set(['b200_sglang', 'mi355x_vllm', 'b200_sglang_mtp'])); + expect(resolved.keptGroup).toBe('sglang'); + expect(resolved.droppedGroups).toEqual(['vllm']); + }); + it('blocks cross-engine adds across official and overlay namespaces', () => { const prev = new Set(['overlay:b300_sglang']); const all = new Set(['overlay:b300_sglang', 'b300_vllm']); @@ -353,6 +416,27 @@ describe('effectiveLegendItems', () => { expect([...out].toSorted()).toEqual(['h100_dynamo-vllm_mtp', 'h100_vllm', 'h100_vllm_mtp']); }); + it('keeps idle hardware scopes in the restore-all universe', () => { + const all = new Set(['b200_sglang', 'b200_vllm', 'mi355x_vllm']); + const active = new Set(['b200_sglang']); + const effective = effectiveLegendItems(all, active, agenticEx); + + expect(effective).toEqual(new Set(['b200_sglang', 'mi355x_vllm'])); + expect(computeToggle(active, 'b200_sglang', effective)).toEqual(effective); + }); + + it('restores the remembered engine for a temporarily idle hardware scope', () => { + const all = new Set(['b200_sglang', 'mi355x_sglang', 'mi355x_vllm']); + const preferred = new Set(['b200_sglang', 'mi355x_vllm']); + const initialUniverse = effectiveLegendItems(all, preferred, agenticEx, preferred); + const solo = computeToggle(preferred, 'b200_sglang', initialUniverse); + const restoreUniverse = effectiveLegendItems(all, solo, agenticEx, preferred); + + expect(solo).toEqual(new Set(['b200_sglang'])); + expect(restoreUniverse).toEqual(preferred); + expect(computeToggle(solo, 'b200_sglang', restoreUniverse)).toEqual(preferred); + }); + it('makes computeToggle solo on click in the default-deselected state', () => { // Default DSv4 state: all non-MTP active, MTP keys exist in data but // are deselected. The effective universe matches active → computeToggle diff --git a/packages/app/src/lib/exclusion.ts b/packages/app/src/lib/exclusion.ts index 86bb5445..21f4d1bd 100644 --- a/packages/app/src/lib/exclusion.ts +++ b/packages/app/src/lib/exclusion.ts @@ -39,6 +39,11 @@ export interface ExclusionSpec { * own group. (e.g. `{ atom: 'sglang' }` — ATOM and SGLang are comparable.) */ groupAliases?: Record; + /** + * Restrict mutual exclusion to configs on the same hardware SKU. Different + * hardware may use different engine groups on the same graph. + */ + scope?: 'hardware'; } /** Compiled resolvers for model- or sequence-scoped exclusion specs. */ @@ -47,12 +52,16 @@ export interface Exclusion { familyOf: (hwKey: string) => string | null; /** Comparability-group id of a participating key (for exclusion), else null. */ groupOf: (hwKey: string) => string | null; + /** Mutual-exclusion scopes of a participating key. Empty when it does not participate. */ + scopesOf: (hwKey: string) => readonly string[]; } const ACTIVE_SPEC_SUFFIXES = [...SPEC_METHOD_KEYS] .filter((method) => method !== 'none') .map((method) => `_${method}`); +const GLOBAL_SCOPE = '*'; + /** * Extract the literal engine family for `hwKey` under a single spec: strip the * configured variant suffix (or require an unsuffixed STP key), drop the leading @@ -81,7 +90,7 @@ function familyForSpec(hwKey: string, spec: ExclusionSpec): string | null { } /** - * Compile a list of `ExclusionSpec`s into `familyOf` / `groupOf` resolvers. + * Compile a list of `ExclusionSpec`s into family, group, and scope resolvers. * The first spec that matches a key wins; variant-specific suffixes and the * unsuffixed STP matcher are disjoint. */ @@ -101,19 +110,40 @@ export function buildExclusion(specs: readonly ExclusionSpec[]): Exclusion { } return null; }, + scopesOf(hwKey: string): readonly string[] { + for (const spec of specs) { + const fam = familyForSpec(hwKey, spec); + if (!fam) continue; + const firstUnderscore = hwKey.indexOf('_'); + const hardwareScope = + firstUnderscore === -1 ? GLOBAL_SCOPE : hwKey.slice(0, firstUnderscore); + return spec.scope === 'hardware' ? [hardwareScope] : [GLOBAL_SCOPE, hardwareScope]; + } + return []; + }, }; } -function groupKeysByGroup(keys: Iterable, ex: Exclusion): Map { - const byGroup = new Map(); +function groupKeysByScope( + keys: Iterable, + ex: Exclusion, +): Map> { + const byScope = new Map>(); for (const key of keys) { const group = ex.groupOf(key); if (!group) continue; - const existing = byGroup.get(group); - if (existing) existing.push(key); - else byGroup.set(group, [key]); + for (const scope of ex.scopesOf(key)) { + let byGroup = byScope.get(scope); + if (!byGroup) { + byGroup = new Map(); + byScope.set(scope, byGroup); + } + const existing = byGroup.get(group); + if (existing) existing.push(key); + else byGroup.set(group, [key]); + } } - return byGroup; + return byScope; } /** @@ -125,22 +155,24 @@ function groupKeysByGroup(keys: Iterable, ex: Exclusion): Map, - group: string | null, + group: string, + scope: string, ex: Exclusion, ): string | null { - if (!group) return null; const families: string[] = []; for (const key of keys) { const fam = ex.familyOf(key); - if (fam && ex.groupOf(key) === group) families.push(fam); + if (fam && ex.groupOf(key) === group && ex.scopesOf(key).includes(scope)) families.push(fam); } return families.length > 0 ? families.toSorted()[0] : null; } /** * Pick a single comparability group to keep when `proposed` contains keys from - * multiple groups. Sticks to a group already present in `prev`; otherwise falls - * back to the alphabetically-first group. Drops other groups' participating keys. + * multiple groups. Sticks to a group already present in `prev`; for a shared + * scope with no direct prior key (for example the global MTP scope), also honors + * a prior key from the same group on an overlapping hardware scope. Otherwise + * falls back to the alphabetically-first group. * * If `proposed` has 0 or 1 groups, the input set is returned unchanged. */ @@ -149,53 +181,110 @@ export function pickStickyGroup( prev: Set, ex: Exclusion, ): { result: Set; keptGroup: string | null; droppedGroups: string[] } { - const byGroup = groupKeysByGroup(proposed, ex); - if (byGroup.size <= 1) { - return { - result: proposed, - keptGroup: byGroup.size === 1 ? [...byGroup.keys()][0] : null, - droppedGroups: [], - }; - } - const prevGroups = new Set(); - for (const key of prev) { - const group = ex.groupOf(key); - if (group) prevGroups.add(group); - } - const groups = [...byGroup.keys()]; - const sticky = groups.find((g) => prevGroups.has(g)); - const winner = sticky ?? [...groups].toSorted()[0]; + const byScope = groupKeysByScope(proposed, ex); + const allGroups = new Set([...byScope.values()].flatMap((byGroup) => [...byGroup.keys()])); const result = new Set(proposed); - const dropped: string[] = []; - for (const [group, keys] of byGroup) { - if (group === winner) continue; - for (const k of keys) result.delete(k); - dropped.push(group); + const winners = new Set(); + const dropped = new Set(); + + for (const [scope, byGroup] of byScope) { + if (byGroup.size <= 1) continue; + const directPrevGroups = new Set(); + for (const key of prev) { + if (!ex.scopesOf(key).includes(scope)) continue; + const group = ex.groupOf(key); + if (group) directPrevGroups.add(group); + } + const groups = [...byGroup.keys()]; + const correlatedPrevGroups = new Set(); + if (directPrevGroups.size === 0) { + for (const [group, keys] of byGroup) { + const relatedScopes = new Set( + [...keys].flatMap((key) => ex.scopesOf(key).filter((candidate) => candidate !== scope)), + ); + if ( + [...prev].some( + (key) => + ex.groupOf(key) === group && + ex.scopesOf(key).some((candidate) => relatedScopes.has(candidate)), + ) + ) { + correlatedPrevGroups.add(group); + } + } + } + const winner = + groups.filter((group) => directPrevGroups.has(group)).toSorted()[0] ?? + groups.filter((group) => correlatedPrevGroups.has(group)).toSorted()[0] ?? + groups.toSorted()[0]; + winners.add(winner); + for (const [group, keys] of byGroup) { + if (group === winner) continue; + for (const key of keys) result.delete(key); + dropped.add(group); + } } - return { result, keptGroup: winner, droppedGroups: dropped }; + + return { + result: dropped.size === 0 ? proposed : result, + keptGroup: + winners.size === 1 + ? [...winners][0] + : winners.size === 0 && allGroups.size === 1 + ? [...allGroups][0] + : null, + droppedGroups: [...dropped], + }; } /** * Compute the effective legend universe for solo/restore-all toggle semantics - * under exclusion. Participating keys whose group is not currently active are - * dropped, so the default-deselected state (e.g. DSv4 MTP on first load) counts - * as "all selected" — clicking an entry then solos it instead of just removing - * it. + * under exclusion. Participating keys whose group is not active or remembered + * for one of their scopes are dropped. Remembered groups preserve each + * hardware's selection while it is temporarily absent in solo mode; an idle + * global scope without a remembered selection remains excluded so default- + * deselected variants (e.g. DSv4 MTP) still count as deselected. */ export function effectiveLegendItems( allItems: Set, active: Set, ex: Exclusion, + preferred: Set = active, ): Set { - const activeGroups = new Set(); - for (const k of active) { - const group = ex.groupOf(k); - if (group) activeGroups.add(group); + const activeGroupsByScope = new Map>(); + for (const key of active) { + const group = ex.groupOf(key); + if (!group) continue; + for (const scope of ex.scopesOf(key)) { + const groups = activeGroupsByScope.get(scope); + if (groups) groups.add(group); + else activeGroupsByScope.set(scope, new Set([group])); + } + } + const preferredGroupsByScope = new Map>(); + for (const key of preferred) { + const group = ex.groupOf(key); + if (!group) continue; + for (const scope of ex.scopesOf(key)) { + const groups = preferredGroupsByScope.get(scope); + if (groups) groups.add(group); + else preferredGroupsByScope.set(scope, new Set([group])); + } } const result = new Set(); - for (const k of allItems) { - const group = ex.groupOf(k); - if (!group || activeGroups.has(group)) result.add(k); + for (const key of allItems) { + const group = ex.groupOf(key); + const scopes = ex.scopesOf(key); + const effectiveScopeGroups = scopes + .map((scope) => activeGroupsByScope.get(scope) ?? preferredGroupsByScope.get(scope)) + .filter((groups): groups is Set => groups !== undefined); + const idleGlobalScope = + scopes.includes(GLOBAL_SCOPE) && + !activeGroupsByScope.has(GLOBAL_SCOPE) && + !preferredGroupsByScope.has(GLOBAL_SCOPE); + if (!group || (!idleGlobalScope && effectiveScopeGroups.every((groups) => groups.has(group)))) { + result.add(key); + } } return result; } @@ -211,15 +300,20 @@ export function clearAllExclusionGroups( proposed: Set, ex: Exclusion, ): { result: Set; droppedGroups: string[] } { - const byGroup = groupKeysByGroup(proposed, ex); - if (byGroup.size <= 1) { - return { result: proposed, droppedGroups: [] }; - } + const byScope = groupKeysByScope(proposed, ex); const result = new Set(proposed); - for (const keys of byGroup.values()) { - for (const k of keys) result.delete(k); + const dropped = new Set(); + for (const byGroup of byScope.values()) { + if (byGroup.size <= 1) continue; + for (const [group, keys] of byGroup) { + for (const key of keys) result.delete(key); + dropped.add(group); + } } - return { result, droppedGroups: [...byGroup.keys()] }; + return { + result: dropped.size === 0 ? proposed : result, + droppedGroups: [...dropped], + }; } export type ExclusionConflictPolicy = 'clear-all' | 'keep-sticky'; @@ -242,12 +336,12 @@ export function resolveExclusionGroups( return { ...cleared, keptGroup: null }; } -/** Literal engine families retained and removed by an exclusion resolution. */ +/** Engine families wholly retained, wholly removed, or retained only on other scopes. */ export function exclusionResolutionFamilies( proposed: Iterable, result: ReadonlySet, ex: Exclusion, -): { kept: string[]; dropped: string[] } { +): { kept: string[]; dropped: string[]; partial: string[] } { const kept = new Set(); const dropped = new Set(); for (const key of proposed) { @@ -255,7 +349,16 @@ export function exclusionResolutionFamilies( if (!family) continue; (result.has(key) ? kept : dropped).add(family); } - return { kept: [...kept].toSorted(), dropped: [...dropped].toSorted() }; + const partial = new Set([...kept].filter((family) => dropped.has(family))); + for (const family of partial) { + kept.delete(family); + dropped.delete(family); + } + return { + kept: [...kept].toSorted(), + dropped: [...dropped].toSorted(), + partial: [...partial].toSorted(), + }; } /** @@ -287,15 +390,19 @@ export function resolveExclusionToggle( const willBeActive = proposed.has(hw); const newFamily = ex.familyOf(hw); const newGroup = ex.groupOf(hw); + const newScopes = ex.scopesOf(hw); - // Hard-block the explicit ADD that introduces a cross-group conflict. Compare - // on the comparability group (so adding an engine in the already-active group, - // e.g. ATOM alongside SGLang, is allowed), but surface the literal engine - // label in the toast. - if (!wasActive && willBeActive && newGroup) { - const sticky = pickStickyGroup(proposed, prev, ex); - if (sticky.droppedGroups.length > 0 && sticky.keptGroup !== newGroup) { - const existing = activeFamilyInGroup(prev, sticky.keptGroup, ex) ?? sticky.keptGroup; + // Hard-block only an explicit add that introduces a second group in an + // overlapping exclusion scope. Global rules also participate in their + // hardware scope, preserving same-SKU STP/MTP conflicts. + if (!wasActive && willBeActive && newGroup && newScopes.length > 0) { + for (const newScope of newScopes) { + const existingGroup = [...prev] + .filter((key) => ex.scopesOf(key).includes(newScope)) + .map((key) => ex.groupOf(key)) + .find((group) => group !== null && group !== newGroup); + if (!existingGroup) continue; + const existing = activeFamilyInGroup(prev, existingGroup, newScope, ex) ?? existingGroup; return { kind: 'block', attempted: newFamily ?? newGroup, existing }; } } diff --git a/packages/app/timings.json b/packages/app/timings.json index b993384b..29003b6b 100644 --- a/packages/app/timings.json +++ b/packages/app/timings.json @@ -2,155 +2,155 @@ "durations": [ { "spec": "cypress/e2e/agentic-point-time-series.cy.ts", - "duration": 2248 + "duration": 2222 }, { "spec": "cypress/e2e/blog.cy.ts", - "duration": 1073 + "duration": 1052 }, { "spec": "cypress/e2e/compare-per-dollar-redirect.cy.ts", - "duration": 1884 + "duration": 2471 }, { "spec": "cypress/e2e/compare-per-dollar-table.cy.ts", - "duration": 510 + "duration": 657 }, { "spec": "cypress/e2e/compare-precision.cy.ts", - "duration": 1049 + "duration": 1519 }, { "spec": "cypress/e2e/compare-redirect.cy.ts", - "duration": 1537 + "duration": 1726 }, { "spec": "cypress/e2e/compare-spec-decode.cy.ts", - "duration": 798 + "duration": 1123 }, { "spec": "cypress/e2e/compare-table.cy.ts", - "duration": 1260 + "duration": 1460 }, { "spec": "cypress/e2e/csv-export.cy.ts", - "duration": 1679 + "duration": 1883 }, { "spec": "cypress/e2e/custom-user-values.cy.ts", - "duration": 11837 + "duration": 12004 }, { "spec": "cypress/e2e/datasets-distributions.cy.ts", - "duration": 338 + "duration": 387 }, { "spec": "cypress/e2e/datasets-flamegraph-time.cy.ts", - "duration": 313 + "duration": 336 }, { "spec": "cypress/e2e/drill-down-trend.cy.ts", - "duration": 3779 + "duration": 4647 }, { "spec": "cypress/e2e/dropdown-switching.cy.ts", - "duration": 1368 + "duration": 1824 }, { "spec": "cypress/e2e/evaluation-chart.cy.ts", - "duration": 1051 + "duration": 1299 }, { "spec": "cypress/e2e/gpu-compare-agentic-detail.cy.ts", - "duration": 1041 + "duration": 1310 }, { "spec": "cypress/e2e/gpu-power.cy.ts", - "duration": 3402 + "duration": 4516 }, { "spec": "cypress/e2e/gpu-specs.cy.ts", - "duration": 4894 + "duration": 5642 }, { "spec": "cypress/e2e/gradient-labels.cy.ts", - "duration": 3466 + "duration": 3702 }, { "spec": "cypress/e2e/historical-trends.cy.ts", - "duration": 4183 + "duration": 5761 }, { "spec": "cypress/e2e/inference-chart.cy.ts", - "duration": 614 + "duration": 865 }, { "spec": "cypress/e2e/inference-replay.cy.ts", - "duration": 3596 + "duration": 3742 }, { "spec": "cypress/e2e/landing-performance.cy.ts", - "duration": 5602 + "duration": 5744 }, { "spec": "cypress/e2e/line-labels.cy.ts", - "duration": 8824 + "duration": 9171 }, { "spec": "cypress/e2e/measured-power-overlay.cy.ts", - "duration": 1055 + "duration": 1723 }, { "spec": "cypress/e2e/model-architecture.cy.ts", - "duration": 4494 + "duration": 5340 }, { "spec": "cypress/e2e/navigation.cy.ts", - "duration": 2713 + "duration": 3296 }, { "spec": "cypress/e2e/nudge-system.cy.ts", - "duration": 12785 + "duration": 26875 }, { "spec": "cypress/e2e/performance.cy.ts", - "duration": 1538 + "duration": 1733 }, { "spec": "cypress/e2e/reliability-chart.cy.ts", - "duration": 1613 + "duration": 1724 }, { "spec": "cypress/e2e/sanity.cy.ts", - "duration": 782 + "duration": 1042 }, { "spec": "cypress/e2e/speed-overlay.cy.ts", - "duration": 3142 + "duration": 3664 }, { "spec": "cypress/e2e/throughput-calculator.cy.ts", - "duration": 5441 + "duration": 8551 }, { "spec": "cypress/e2e/ttft-x-axis-toggle.cy.ts", - "duration": 1283 + "duration": 2592 }, { "spec": "cypress/e2e/unofficial-watermark.cy.ts", - "duration": 848 + "duration": 1304 }, { "spec": "cypress/e2e/url-params.cy.ts", - "duration": 5910 + "duration": 7335 }, { "spec": "cypress/e2e/yaxis-metrics-render.cy.ts", - "duration": 2690 + "duration": 2990 }, { "spec": "cypress/e2e/zh-pages.cy.ts", - "duration": 1458 + "duration": 1562 } ] }