Skip to content

Commit 29af12d

Browse files
cameroncookecodex
cameroncooke
and
codex
committed
fix: Preserve simulator snapshot test cases
Keep suite-less simulator test cases in structured snapshot fixtures so normalization does not hide output contract changes. Replace the previous fixture-specific progress collapse with shape-based normalization that preserves final counts and rejects malformed progress sequences. Co-Authored-By: OpenAI Codex <noreply@openai.com>
1 parent 4948dad commit 29af12d

5 files changed

Lines changed: 244 additions & 53 deletions

File tree

‎src/snapshot-tests/__fixtures__/json/simulator/test--failure.json‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,176 @@
6565
]
6666
},
6767
"testCases": [
68+
{
69+
"test": "Adding decimal numbers",
70+
"status": "passed",
71+
"durationMs": 0
72+
},
73+
{
74+
"test": "Adding multiple digit numbers",
75+
"status": "passed",
76+
"durationMs": 0
77+
},
78+
{
79+
"test": "Adding single digit numbers",
80+
"status": "passed",
81+
"durationMs": 0
82+
},
83+
{
84+
"test": "Addition operation",
85+
"status": "passed",
86+
"durationMs": 0
87+
},
88+
{
89+
"test": "Calculate without setting operation",
90+
"status": "passed",
91+
"durationMs": 0
92+
},
93+
{
94+
"test": "Calculator handles invalid input gracefully",
95+
"status": "passed",
96+
"durationMs": 0
97+
},
98+
{
99+
"test": "Calculator initializes with correct default values",
100+
"status": "passed",
101+
"durationMs": 0
102+
},
103+
{
104+
"test": "Calculator state after multiple clears",
105+
"status": "passed",
106+
"durationMs": 0
107+
},
108+
{
109+
"test": "Chain calculations",
110+
"status": "passed",
111+
"durationMs": 0
112+
},
113+
{
114+
"test": "Clear function resets calculator to initial state",
115+
"status": "passed",
116+
"durationMs": 0
117+
},
118+
{
119+
"test": "Clear input through handler",
120+
"status": "passed",
121+
"durationMs": 0
122+
},
123+
{
124+
"test": "Complex calculation sequence",
125+
"status": "passed",
126+
"durationMs": 0
127+
},
128+
{
129+
"test": "Decimal input through handler",
130+
"status": "passed",
131+
"durationMs": 0
132+
},
133+
{
134+
"test": "Decimal operations precision",
135+
"status": "passed",
136+
"durationMs": 0
137+
},
138+
{
139+
"test": "Decimal point at start creates 0.",
140+
"status": "passed",
141+
"durationMs": 0
142+
},
143+
{
144+
"test": "Division by zero returns zero",
145+
"status": "passed",
146+
"durationMs": 0
147+
},
148+
{
149+
"test": "Division operation",
150+
"status": "passed",
151+
"durationMs": 0
152+
},
153+
{
154+
"test": "Expression display updates correctly",
155+
"status": "passed",
156+
"durationMs": 0
157+
},
158+
{
159+
"test": "Large number error handling",
160+
"status": "passed",
161+
"durationMs": 0
162+
},
163+
{
164+
"test": "Multiple decimal points should be ignored",
165+
"status": "passed",
166+
"durationMs": 0
167+
},
168+
{
169+
"test": "Multiple equals presses",
170+
"status": "passed",
171+
"durationMs": 0
172+
},
173+
{
174+
"test": "Multiplication operation",
175+
"status": "passed",
176+
"durationMs": 0
177+
},
178+
{
179+
"test": "Number input through handler",
180+
"status": "passed",
181+
"durationMs": 0
182+
},
183+
{
184+
"test": "Operation input through handler",
185+
"status": "passed",
186+
"durationMs": 0
187+
},
188+
{
189+
"test": "Percentage calculation",
190+
"status": "passed",
191+
"durationMs": 0
192+
},
193+
{
194+
"test": "Repetitive equals press repeats last operation",
195+
"status": "passed",
196+
"durationMs": 0
197+
},
198+
{
199+
"test": "Setting operation without previous number",
200+
"status": "passed",
201+
"durationMs": 0
202+
},
203+
{
204+
"test": "Simple addition calculation",
205+
"status": "passed",
206+
"durationMs": 0
207+
},
208+
{
209+
"test": "Subtraction operation",
210+
"status": "passed",
211+
"durationMs": 0
212+
},
68213
{
69214
"test": "This test should fail to verify error reporting",
70215
"status": "failed",
71216
"durationMs": 0
72217
},
218+
{
219+
"test": "Toggle sign on negative number",
220+
"status": "passed",
221+
"durationMs": 0
222+
},
223+
{
224+
"test": "Toggle sign on positive number",
225+
"status": "passed",
226+
"durationMs": 0
227+
},
228+
{
229+
"test": "Toggle sign on zero has no effect",
230+
"status": "passed",
231+
"durationMs": 0
232+
},
233+
{
234+
"test": "Very small decimal numbers",
235+
"status": "passed",
236+
"durationMs": 0
237+
},
73238
{
74239
"suite": "CalculatorAppTests",
75240
"test": "testAddition",

‎src/snapshot-tests/__tests__/json-normalize.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import type { StructuredOutputEnvelope } from '../../types/structured-output.ts'
33
import { normalizeStructuredEnvelope } from '../json-normalize.ts';
44

55
describe('normalizeStructuredEnvelope', () => {
6-
it('normalizes volatile simulator Swift Testing passed test cases without dropping failures', () => {
6+
it('keeps suite-less simulator test cases while normalizing volatile durations', () => {
77
const envelope: StructuredOutputEnvelope<unknown> = {
88
schema: 'xcodebuildmcp.output.test-result',
99
schemaVersion: '1',
@@ -28,6 +28,7 @@ describe('normalizeStructuredEnvelope', () => {
2828
summary: { target: 'simulator' },
2929
testCases: [
3030
{ test: 'Swift Testing failure', status: 'failed', durationMs: 0 },
31+
{ test: 'Volatile Swift Testing pass', status: 'passed', durationMs: 0 },
3132
{ suite: 'XCTestSuite', test: 'testStablePass', status: 'passed', durationMs: 0 },
3233
],
3334
},

‎src/snapshot-tests/__tests__/normalize.test.ts‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ function progressBlock(total: number, failed: number): string {
1010
}
1111

1212
describe('normalizeSnapshotOutput', () => {
13-
it('collapses the long simulator failure progress stream while preserving final counts', () => {
14-
const normalized = normalizeSnapshotOutput(`${progressBlock(57, 3)}\n`);
13+
it('collapses long simulator failure progress streams while preserving final counts', () => {
14+
const normalized = normalizeSnapshotOutput(`${progressBlock(42, 3)}\n`);
1515

1616
expect(normalized).toBe(
17-
'Running tests (<TEST_PROGRESS>; final: 57 completed, 3 failed, 0 skipped)\n',
17+
'Running tests (<TEST_PROGRESS>; final: 42 completed, 3 failed, 0 skipped)\n',
1818
);
1919
});
2020

@@ -24,9 +24,29 @@ describe('normalizeSnapshotOutput', () => {
2424
expect(normalizeSnapshotOutput(block)).toBe(block);
2525
});
2626

27-
it('does not collapse unrelated long progress streams', () => {
28-
const block = `${progressBlock(40, 2)}\n`;
27+
it('does not collapse long successful progress streams', () => {
28+
const block = `${progressBlock(40, 0)}\n`;
2929

3030
expect(normalizeSnapshotOutput(block)).toBe(block);
3131
});
32+
33+
it('collapses long simulator failure progress streams that start after the initial zero update', () => {
34+
const normalized = normalizeSnapshotOutput(
35+
`${progressBlock(42, 3).split('\n').slice(1).join('\n')}\n`,
36+
);
37+
38+
expect(normalized).toBe(
39+
'Running tests (<TEST_PROGRESS>; final: 42 completed, 3 failed, 0 skipped)\n',
40+
);
41+
});
42+
43+
it('does not collapse progress streams with non-monotonic counts', () => {
44+
const block = [
45+
progressBlock(20, 0),
46+
'Running tests (19 completed, 0 failures, 0 skipped)',
47+
progressBlock(40, 2).split('\n').slice(21).join('\n'),
48+
].join('\n');
49+
50+
expect(normalizeSnapshotOutput(`${block}\n`)).toBe(`${block}\n`);
51+
});
3252
});

‎src/snapshot-tests/json-normalize.ts‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -92,45 +92,10 @@ function normalizeValue(value: unknown, path: string[] = []): unknown {
9292
return value;
9393
}
9494

95-
function isSimulatorTestResultEnvelope(envelope: StructuredOutputEnvelope<unknown>): boolean {
96-
if (envelope.schema !== 'xcodebuildmcp.output.test-result' || !isRecord(envelope.data)) {
97-
return false;
98-
}
99-
100-
const summary = envelope.data.summary;
101-
return isRecord(summary) && summary.target === 'simulator';
102-
}
103-
104-
function isSuiteLessPassedTestCase(value: unknown): boolean {
105-
return isRecord(value) && value.suite === undefined && value.status === 'passed';
106-
}
107-
108-
function normalizeSimulatorTestCases(
109-
envelope: StructuredOutputEnvelope<unknown>,
110-
): StructuredOutputEnvelope<unknown> {
111-
if (!isSimulatorTestResultEnvelope(envelope) || !isRecord(envelope.data)) {
112-
return envelope;
113-
}
114-
115-
const testCases = envelope.data.testCases;
116-
if (!Array.isArray(testCases)) {
117-
return envelope;
118-
}
119-
120-
return {
121-
...envelope,
122-
data: {
123-
...envelope.data,
124-
testCases: testCases.filter((testCase) => !isSuiteLessPassedTestCase(testCase)),
125-
},
126-
};
127-
}
128-
12995
export function normalizeStructuredEnvelope(
13096
envelope: StructuredOutputEnvelope<unknown>,
13197
): StructuredOutputEnvelope<unknown> {
132-
const normalized = normalizeValue(envelope) as StructuredOutputEnvelope<unknown>;
133-
return normalizeSimulatorTestCases(normalized);
98+
return normalizeValue(envelope) as StructuredOutputEnvelope<unknown>;
13499
}
135100

136101
function compactFrameObjects(json: string): string {

‎src/snapshot-tests/normalize.ts‎

Lines changed: 51 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,61 @@ const ACQUIRED_USAGE_ASSERTION_TIME_REGEX =
4545
const BUILD_SETTINGS_PATH_REGEX = /^( {6}PATH = ).+$/gm;
4646
const TRAILING_WHITESPACE_REGEX = /[ \t]+$/gm;
4747
const SIMULATOR_FAILURE_TEST_PROGRESS_BLOCK_REGEX =
48-
/(?:^Running tests \((\d+) completed, (\d+) failures?, (\d+) skipped\)\n){30,}/gm;
48+
/(?:^Running tests \(\d+ completed, \d+ failures?, \d+ skipped\)\n){30,}/gm;
49+
const TEST_PROGRESS_LINE_REGEX =
50+
/^Running tests \((\d+) completed, (\d+) failures?, (\d+) skipped\)$/u;
51+
52+
type TestProgress = { completed: number; failed: number; skipped: number };
4953

5054
function escapeRegex(str: string): string {
5155
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
5256
}
5357

58+
function parseTestProgressLine(line: string): TestProgress | null {
59+
const match = line.match(TEST_PROGRESS_LINE_REGEX);
60+
if (!match) {
61+
return null;
62+
}
63+
64+
return {
65+
completed: Number(match[1]),
66+
failed: Number(match[2]),
67+
skipped: Number(match[3]),
68+
};
69+
}
70+
71+
function isMonotonicProgress(progress: TestProgress[]): boolean {
72+
return progress.every((current, index) => {
73+
const previous = progress[index - 1];
74+
return (
75+
previous === undefined ||
76+
(current.completed >= previous.completed &&
77+
current.failed >= previous.failed &&
78+
current.skipped >= previous.skipped)
79+
);
80+
});
81+
}
82+
83+
function normalizeSimulatorFailureTestProgressBlock(match: string): string {
84+
const progress = match.trimEnd().split('\n').map(parseTestProgressLine);
85+
const parsedProgress = progress.filter((line): line is TestProgress => line !== null);
86+
if (parsedProgress.length !== progress.length) {
87+
return match;
88+
}
89+
const first = parsedProgress[0];
90+
const final = parsedProgress.at(-1);
91+
if (!first || !final) {
92+
return match;
93+
}
94+
95+
const hasCleanStart = first.completed <= 1 && first.failed === 0 && first.skipped === 0;
96+
if (!hasCleanStart || final.failed === 0 || !isMonotonicProgress(parsedProgress)) {
97+
return match;
98+
}
99+
100+
return `Running tests (<TEST_PROGRESS>; final: ${final.completed} completed, ${final.failed} failed, ${final.skipped} skipped)\n`;
101+
}
102+
54103
export function normalizeSnapshotOutput(text: string): string {
55104
let normalized = text;
56105

@@ -142,16 +191,7 @@ export function normalizeSnapshotOutput(text: string): string {
142191

143192
normalized = normalized.replace(
144193
SIMULATOR_FAILURE_TEST_PROGRESS_BLOCK_REGEX,
145-
(match: string, completed: string, failed: string, skipped: string) => {
146-
if (
147-
!match.startsWith('Running tests (0 completed, 0 failures, 0 skipped)\n') ||
148-
completed !== '57'
149-
) {
150-
return match;
151-
}
152-
153-
return `Running tests (<TEST_PROGRESS>; final: ${completed} completed, ${failed} failed, ${skipped} skipped)\n`;
154-
},
194+
normalizeSimulatorFailureTestProgressBlock,
155195
);
156196

157197
// Normalize final test summary line (counts vary across environments)

0 commit comments

Comments
 (0)