Skip to content

Commit 542381b

Browse files
AmirK-Sclaudepcarleton
authored
feat(tools-list): check deterministic tools/list ordering on the 2026-07-28 wire (#492)
* feat(tools-list): check deterministic tools/list ordering on the 2026-07-28 wire The 2026-07-28 revision adds to server/tools.mdx: "Servers SHOULD return tools in a deterministic order (i.e., the same ordering across requests when the underlying set of tools has not changed)." Nothing in the suite exercised it. Add a tools-list-deterministic-order check to the existing tools-list scenario. From the 2026-07-28 wire on, the scenario takes two more consecutive tools/list snapshots and compares the three name sequences. Same multiset of names, different order is WARNING (SHOULD). A set that changed between probes is reported as untestable rather than as a violation, because the spec scopes the SHOULD to an unchanged set. Fewer than two tools on every probe is INFO. The check carries source.introducedIn '2026-07-28', written as a literal since that revision is published and dated, so runners drop it on earlier wires. Prove it fails: examples/servers/typescript/tools-list-rotated-order.ts advertises the same four tools and rotates the list by one position per call; a negative.test.ts case asserts WARNING while tools-list itself still passes. Unit tests cover the helper that builds the check. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BjeC3sCvfHRpdEPz2x41B2 * tools-list: inline the deterministic-order check id and introducedIn literal Check ids and introducedIn revisions are written inline everywhere else in src/scenarios (tests pin the literal slug on purpose, since SDK baselines reference it), so drop the two exported constants. Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com> Co-authored-by: Paul Carleton <paulc@anthropic.com>
1 parent 0d824ae commit 542381b

4 files changed

Lines changed: 416 additions & 2 deletions

File tree

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* tools/list ordering negative test server.
5+
*
6+
* Speaks the sessionless 2026-07-28 wire (SEP-2575) and advertises the same
7+
* four tools on every tools/list request, but rotates the list by one
8+
* position each time (it keeps a call counter for that, nothing else). The set never changes, only the order, which violates the
9+
* 2026-07-28 SHOULD "Servers SHOULD return tools in a deterministic order".
10+
* The tools-list scenario should emit WARNING for
11+
* tools-list-deterministic-order against this server while tools-list itself
12+
* still passes, since every response is structurally valid.
13+
*/
14+
15+
import express from 'express';
16+
17+
const app = express();
18+
app.use(express.json());
19+
20+
const TOOLS = ['alpha', 'bravo', 'charlie', 'delta'].map((name) => ({
21+
name,
22+
description: `Fixture tool ${name}`,
23+
inputSchema: { type: 'object', properties: {} }
24+
}));
25+
26+
let listCalls = 0;
27+
28+
app.post('/mcp', (req, res) => {
29+
const body = req.body || {};
30+
const id = body.id ?? null;
31+
const method = body.method;
32+
33+
switch (method) {
34+
case 'server/discover':
35+
return res.json({
36+
jsonrpc: '2.0',
37+
id,
38+
result: {
39+
resultType: 'complete',
40+
ttlMs: 0,
41+
cacheScope: 'private',
42+
supportedVersions: ['2026-07-28'],
43+
capabilities: { tools: {} },
44+
serverInfo: { name: 'tools-list-rotated-order', version: '1.0.0' }
45+
}
46+
});
47+
case 'tools/list': {
48+
// Rotate by one position per call: the same set, never the same order.
49+
const offset = listCalls++ % TOOLS.length;
50+
const tools = [...TOOLS.slice(offset), ...TOOLS.slice(0, offset)];
51+
return res.json({
52+
jsonrpc: '2.0',
53+
id,
54+
result: {
55+
resultType: 'complete',
56+
ttlMs: 0,
57+
cacheScope: 'private',
58+
tools
59+
}
60+
});
61+
}
62+
default:
63+
return res.status(404).json({
64+
jsonrpc: '2.0',
65+
id,
66+
error: { code: -32601, message: 'Method not found' }
67+
});
68+
}
69+
});
70+
71+
const PORT = parseInt(process.env.PORT || '3008', 10);
72+
app.listen(PORT, '127.0.0.1', () => {
73+
console.log(
74+
`tools/list rotated-order negative test server running on http://localhost:${PORT}/mcp`
75+
);
76+
});

‎src/scenarios/server/negative.test.ts‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ import path from 'path';
44
import { DNSRebindingProtectionScenario } from './dns-rebinding';
55
import { ResourcesNotFoundErrorScenario } from './resources';
66
import { CachingScenario } from './caching';
7+
import { ToolsListScenario } from './tools';
78
import {
89
JsonSchema2020_12Scenario,
910
sep2106KeywordCheckStatus
1011
} from './json-schema-2020-12';
11-
import { ToolsListScenario } from './tools';
1212
import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../../types';
1313
import { takeWireViolations } from '../../validation/wire-schema';
1414

@@ -217,6 +217,43 @@ describe('Server scenario negative tests', () => {
217217
}, 10000);
218218
});
219219

220+
describe('tools-list-deterministic-order', () => {
221+
let serverProcess: ChildProcess | null = null;
222+
const PORT = 3008;
223+
224+
beforeAll(async () => {
225+
serverProcess = await startServer(
226+
path.join(
227+
process.cwd(),
228+
'examples/servers/typescript/tools-list-rotated-order.ts'
229+
),
230+
PORT
231+
);
232+
}, 35000);
233+
234+
afterAll(async () => {
235+
await stopServer(serverProcess);
236+
});
237+
238+
it('emits WARNING for deterministic-order while tools-list still passes against a server that rotates its tool list', async () => {
239+
const scenario = new ToolsListScenario();
240+
const checks = await scenario.run(
241+
testContext(`http://localhost:${PORT}/mcp`, DRAFT_PROTOCOL_VERSION)
242+
);
243+
244+
const list = checks.find((c) => c.id === 'tools-list');
245+
expect(list?.status).toBe('SUCCESS');
246+
247+
const order = checks.find(
248+
(c) => c.id === 'tools-list-deterministic-order'
249+
);
250+
expect(order?.status).toBe('WARNING');
251+
expect(order?.errorMessage).toMatch(/different order/);
252+
expect(order?.details).toMatchObject({ toolCount: 4, probes: 3 });
253+
expect(order?.details?.untestable).toBeUndefined();
254+
}, 10000);
255+
});
256+
220257
describe('tools-name-format', () => {
221258
let serverProcess: ChildProcess | null = null;
222259
const PORT = 3009;

‎src/scenarios/server/tools.test.ts‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { describe, it, expect } from 'vitest';
22
import {
3+
buildToolsListDeterministicOrderCheck,
34
buildToolsNameFormatCheck,
45
toolNameFormatCheckApplies,
56
ToolsListScenario,
@@ -193,3 +194,143 @@ describe('ToolsListScenario version gate', () => {
193194
);
194195
});
195196
});
197+
198+
describe('buildToolsListDeterministicOrderCheck', () => {
199+
const names = (...ns: string[]) => ns.map((name) => ({ name }));
200+
201+
it('returns SUCCESS when every probe lists the same tools in the same order', () => {
202+
const check = buildToolsListDeterministicOrderCheck([
203+
names('a', 'b', 'c'),
204+
names('a', 'b', 'c'),
205+
names('a', 'b', 'c')
206+
]);
207+
expect(check.id).toBe('tools-list-deterministic-order');
208+
expect(check.status).toBe('SUCCESS');
209+
expect(check.errorMessage).toBeUndefined();
210+
expect(check.details).toEqual({
211+
toolCount: 3,
212+
probes: 3,
213+
orders: [
214+
['a', 'b', 'c'],
215+
['a', 'b', 'c'],
216+
['a', 'b', 'c']
217+
]
218+
});
219+
expect(check.specReferences?.[0]?.url).toContain('2026-07-28/server/tools');
220+
});
221+
222+
it('returns WARNING when the same tools come back in a different order', () => {
223+
const check = buildToolsListDeterministicOrderCheck([
224+
names('a', 'b', 'c'),
225+
names('b', 'c', 'a'),
226+
names('c', 'a', 'b')
227+
]);
228+
expect(check.status).toBe('WARNING');
229+
expect(check.errorMessage).toMatch(/different order/);
230+
expect(check.errorMessage).toMatch(/index 0/);
231+
expect(check.details).toMatchObject({ toolCount: 3, probes: 3 });
232+
expect(check.details?.untestable).toBeUndefined();
233+
expect(check.details?.orders).toEqual([
234+
['a', 'b', 'c'],
235+
['b', 'c', 'a'],
236+
['c', 'a', 'b']
237+
]);
238+
});
239+
240+
it('flags a divergence that only appears on the last probe', () => {
241+
const check = buildToolsListDeterministicOrderCheck([
242+
names('a', 'b', 'c'),
243+
names('a', 'b', 'c'),
244+
names('a', 'c', 'b')
245+
]);
246+
expect(check.status).toBe('WARNING');
247+
expect(check.errorMessage).toMatch(/probe 3/);
248+
expect(check.errorMessage).toMatch(/index 1/);
249+
});
250+
251+
it('reports untestable (WARNING) when the set of tools changed between probes', () => {
252+
const check = buildToolsListDeterministicOrderCheck([
253+
names('a', 'b', 'c'),
254+
names('a', 'b', 'd')
255+
]);
256+
expect(check.status).toBe('WARNING');
257+
expect(check.errorMessage).toMatch(/^Not testable: /);
258+
expect(check.errorMessage).toMatch(/added: d/);
259+
expect(check.errorMessage).toMatch(/removed: c/);
260+
expect(check.details).toMatchObject({ untestable: true });
261+
});
262+
263+
it('reports untestable (WARNING) when a probe returned no tools array', () => {
264+
const check = buildToolsListDeterministicOrderCheck([
265+
names('a', 'b'),
266+
undefined
267+
]);
268+
expect(check.status).toBe('WARNING');
269+
expect(check.errorMessage).toMatch(/^Not testable: /);
270+
expect(check.details).toMatchObject({ untestable: true });
271+
});
272+
273+
it('reports untestable (WARNING) with fewer than two probes', () => {
274+
const check = buildToolsListDeterministicOrderCheck([names('a', 'b')]);
275+
expect(check.status).toBe('WARNING');
276+
expect(check.errorMessage).toMatch(/^Not testable: /);
277+
});
278+
279+
it('returns INFO when no probe saw two tools to order', () => {
280+
const one = buildToolsListDeterministicOrderCheck([names('a'), names('a')]);
281+
expect(one.status).toBe('INFO');
282+
expect(one.errorMessage).toMatch(/nothing to compare/);
283+
expect(one.details).toEqual({
284+
toolCount: 1,
285+
probes: 2,
286+
orders: [['a'], ['a']]
287+
});
288+
const none = buildToolsListDeterministicOrderCheck([[], []]);
289+
expect(none.status).toBe('INFO');
290+
});
291+
292+
it('reports untestable when a probe grows from one tool to two', () => {
293+
const check = buildToolsListDeterministicOrderCheck([
294+
names('a'),
295+
names('a', 'b')
296+
]);
297+
expect(check.status).toBe('WARNING');
298+
expect(check.errorMessage).toMatch(/^Not testable: /);
299+
expect(check.errorMessage).toMatch(/added: b/);
300+
});
301+
302+
it('treats a tool without a string name as one placeholder entry', () => {
303+
const check = buildToolsListDeterministicOrderCheck([
304+
[{ name: 'a' }, { name: 42 }],
305+
[{ name: 42 }, { name: 'a' }]
306+
]);
307+
expect(check.status).toBe('WARNING');
308+
expect(check.errorMessage).toMatch(/different order/);
309+
expect(check.details?.orders).toEqual([
310+
['a', '<tool missing name>'],
311+
['<tool missing name>', 'a']
312+
]);
313+
});
314+
315+
it('treats duplicate names as a set change only when their multiplicity changes', () => {
316+
const stable = buildToolsListDeterministicOrderCheck([
317+
names('a', 'a', 'b'),
318+
names('a', 'a', 'b')
319+
]);
320+
expect(stable.status).toBe('SUCCESS');
321+
const changed = buildToolsListDeterministicOrderCheck([
322+
names('a', 'a', 'b'),
323+
names('a', 'b', 'b')
324+
]);
325+
expect(changed.status).toBe('WARNING');
326+
expect(changed.errorMessage).toMatch(/^Not testable: /);
327+
});
328+
329+
it('gates itself to the 2026-07-28 wire', () => {
330+
const check = buildToolsListDeterministicOrderCheck([
331+
names('a', 'b'),
332+
names('a', 'b')
333+
]);
334+
expect(check.source).toEqual({ introducedIn: '2026-07-28' });
335+
});
336+
});

0 commit comments

Comments
 (0)