Skip to content

Commit fb81ab9

Browse files
committed
Merge origin/main (SEP-2640 skills) into claude/hosted-client-conformance
main added the SEP-2640 skills scenarios (#330). Conflict in src/scenarios/index.ts was additive on both sides (hosted branch appended StatelessGauntletScenario/AuthCheckerScenario, main appended the four skills client scenarios); both are kept. SkillsVerificationScenario is one class registered three times with a constructor argument, which the hosted runner's per-run `new (proto.constructor)()` instantiation dropped (the "every hostable scenario can be instantiated" test caught it). Add an optional `Scenario.fresh()` hook, route the three instantiation sites through a shared `freshScenario()` helper that prefers it, implement it on SkillsVerificationScenario, document the contract in src/hosted/README.md, and have the test assert the per-run instance keeps its registry name. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FXWixCiyW8eEfwFeZcADEK
2 parents 8bf9546 + 7169291 commit fb81ab9

17 files changed

Lines changed: 3806 additions & 11 deletions

‎src/hosted/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ Hostable = any scenario that implements `handler()` (single origin) or
4646
`authHandlers()` (multi-origin, see below). `listHostableScenarios()` derives
4747
the list at runtime, gated by which aux origins are configured.
4848

49+
Each run gets its own scenario instance, built from the registry entry with a
50+
no-arg constructor. A scenario whose constructor takes parameters (one class
51+
registered under several names, e.g. `skills/verification-*`) implements
52+
`Scenario.fresh()` to carry them into the per-run copy.
53+
4954
`sse-retry` implements `handler()` and works under `conformance hosted`, but
5055
its connection-close-timing checks won't be meaningful through a buffered
5156
fetch bridge — see below.

‎src/hosted/hosted.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,10 +258,13 @@ describe('hosted server', () => {
258258

259259
it('every hostable scenario can be instantiated without binding a port', () => {
260260
// Guard against regressions where a handler() implementation reaches for
261-
// this._server / this.port etc.
261+
// this._server / this.port etc., and against per-run instances that lose
262+
// constructor parameters (one class registered under several names must
263+
// implement fresh()).
262264
for (const name of listHostableScenarios()) {
263265
const run = sessions.getOrCreate(name, `probe-${name}`, () => 'http://x');
264266
expect(typeof run.listener).toBe('function');
267+
expect(run.scenario.name).toBe(name);
265268
}
266269
});
267270
});

‎src/hosted/server.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ import {
3636
UnknownScenarioError,
3737
NotHostableError,
3838
listHostableScenarios,
39-
hostedScenarioContext
39+
hostedScenarioContext,
40+
freshScenario
4041
} from './session';
4142
import { renderLanding, renderResults } from './html';
4243
import type { RunStore } from './store';
@@ -390,8 +391,7 @@ export function createHostedApp(opts: HostedServerOptions = {}): {
390391
): StatelessInstance {
391392
const proto = getScenario(scenarioName);
392393
if (!proto) throw new UnknownScenarioError(scenarioName);
393-
const Ctor = proto.constructor as new () => Scenario;
394-
const scenario = new Ctor();
394+
const scenario = freshScenario(proto);
395395

396396
if (scenario instanceof AuthHandlerScenario) {
397397
const missing = scenario.auxRoles.filter((r) => !auxOrigins[r]);

‎src/hosted/session.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,17 @@ export function rawChecksOf(scenario: Scenario): ConformanceCheck[] {
9999
return scenario.getChecks();
100100
}
101101

102+
/**
103+
* New, unstarted instance of a registered scenario. Prefers `Scenario.fresh()`
104+
* so scenarios registered with constructor parameters keep them; falls back
105+
* to the no-arg constructor.
106+
*/
107+
export function freshScenario(proto: Scenario): Scenario {
108+
if (typeof proto.fresh === 'function') return proto.fresh();
109+
const Ctor = proto.constructor as new () => Scenario;
110+
return new Ctor();
111+
}
112+
102113
/**
103114
* Judge a merged raw log with the scenario's own end-of-run logic by loading
104115
* it into a fresh instance. Falls back to the raw log for scenarios that
@@ -111,8 +122,7 @@ export function finalizeChecks(
111122
const proto = getScenario(scenarioName);
112123
if (!proto) return merged;
113124
try {
114-
const Ctor = proto.constructor as new () => Scenario;
115-
const fresh = new Ctor() as unknown as {
125+
const fresh = freshScenario(proto) as unknown as {
116126
checks?: unknown;
117127
getChecks(): ConformanceCheck[];
118128
};
@@ -165,8 +175,7 @@ export class SessionManager {
165175
const proto = getScenario(scenarioName);
166176
if (!proto) throw new UnknownScenarioError(scenarioName);
167177

168-
const Ctor = proto.constructor as new () => Scenario;
169-
const scenario = new Ctor();
178+
const scenario = freshScenario(proto);
170179
const runId = id ?? randomBytes(6).toString('base64url');
171180

172181
let listener: RequestListener;

‎src/scenarios/client/draft-result-fields.test.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import type { BaseHttpScenario } from './http-base';
1212
import { RequestMetadataScenario } from './request-metadata';
1313
import { MRTRClientScenario } from './mrtr-client';
1414
import { JsonSchemaRefDerefScenario } from './json-schema-ref-deref';
15+
import { SkillsNoPrefetchScenario } from './skills/no-prefetch';
16+
import { SkillsVerificationScenario } from './skills/verification';
1517

1618
/**
1719
* Pins that the hand-rolled mock servers used by client-direction scenarios
@@ -528,3 +530,96 @@ describe('sep-2322-client-request-state mock results (2026-07-28)', () => {
528530
}
529531
});
530532
});
533+
534+
describe('sep-2640 skills client mock results', () => {
535+
const skillsScenarios: Array<{
536+
name: string;
537+
make: () => BaseHttpScenario;
538+
}> = [
539+
{
540+
name: 'sep-2640-client-no-prefetch',
541+
make: () => new SkillsNoPrefetchScenario()
542+
},
543+
{
544+
name: 'sep-2640-client-verify-digest',
545+
make: () => new SkillsVerificationScenario('digest')
546+
},
547+
{
548+
name: 'sep-2640-client-verify-size',
549+
make: () => new SkillsVerificationScenario('size')
550+
},
551+
{
552+
name: 'sep-2640-client-verify-frontmatter',
553+
make: () => new SkillsVerificationScenario('frontmatter')
554+
}
555+
];
556+
557+
for (const s of skillsScenarios) {
558+
it(`${s.name} carries the draft-required members on skills/list and resources/read`, async () => {
559+
const scenario = s.make();
560+
const { serverUrl } = await scenario.start(
561+
testScenarioContext(DRAFT_PROTOCOL_VERSION)
562+
);
563+
try {
564+
let id = 1;
565+
for (const [method, params] of [
566+
['skills/list', {}],
567+
['resources/read', { uri: 'skill://pdf-processing/SKILL.md' }]
568+
] as const) {
569+
const { status, body } = await post(
570+
serverUrl,
571+
{
572+
jsonrpc: '2.0',
573+
id: id++,
574+
method,
575+
params: { ...params, _meta: meta }
576+
},
577+
{
578+
'mcp-protocol-version': DRAFT_PROTOCOL_VERSION,
579+
'Mcp-Method': method
580+
}
581+
);
582+
expect(status, method).toBe(200);
583+
expect(body.result, method).toMatchObject(CACHEABLE_FIELDS);
584+
if (method === 'resources/read') {
585+
expect(
586+
wireSchemaErrors(DRAFT_PROTOCOL_VERSION, body, method),
587+
method
588+
).toEqual([]);
589+
}
590+
}
591+
} finally {
592+
await scenario.stop();
593+
}
594+
});
595+
596+
// The runner hands these scenarios' clients 2025-11-25 unless
597+
// --spec-version says otherwise, and the extension has no 2026-07-28
598+
// dependency, so the mock answers at the version the client asked for.
599+
it(`${s.name} answers initialize at the requested protocol version`, async () => {
600+
const scenario = s.make();
601+
const { serverUrl } = await scenario.start(testScenarioContext());
602+
try {
603+
let id = 1;
604+
for (const protocolVersion of ['2025-11-25', DRAFT_PROTOCOL_VERSION]) {
605+
const { status, body } = await post(serverUrl, {
606+
jsonrpc: '2.0',
607+
id: id++,
608+
method: 'initialize',
609+
params: {
610+
protocolVersion,
611+
capabilities: {},
612+
clientInfo: { name: 'test', version: '1.0' }
613+
}
614+
});
615+
expect(status, protocolVersion).toBe(200);
616+
expect(body.result.protocolVersion, protocolVersion).toBe(
617+
protocolVersion
618+
);
619+
}
620+
} finally {
621+
await scenario.stop();
622+
}
623+
});
624+
}
625+
});
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import { describe, test, expect } from 'vitest';
2+
import { testScenarioContext } from '../../../mock-server/testing';
3+
import { sendStatelessRequest } from '../../../connection/stateless';
4+
import { DRAFT_PROTOCOL_VERSION } from '../../../types';
5+
import { SkillsNoPrefetchScenario } from './no-prefetch';
6+
import { SkillsVerificationScenario } from './verification';
7+
8+
/**
9+
* Regression for PR #330 review: both skills client scenarios declared the
10+
* skills extension only in their `initialize` reply, but `BaseHttpScenario`
11+
* intercepts `server/discover` before `handlePost` runs. A 2026-07-28
12+
* discover-first client therefore saw the base class default (`{tools: {}}`)
13+
* and had nothing to gate a `skills/list` call on.
14+
*
15+
* The scenarios passed anyway because `server/discover` is optional for
16+
* clients and the clients we ran did not use it. That is exactly the failure
17+
* mode where a check goes green because its condition never arose, so both
18+
* lifecycles are asserted here rather than left to the next client to notice.
19+
*/
20+
21+
const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills';
22+
23+
type Caps = {
24+
resources?: unknown;
25+
extensions?: Record<string, unknown>;
26+
};
27+
28+
function capsOf(result: unknown): Caps {
29+
return ((result as { capabilities?: Caps })?.capabilities ?? {}) as Caps;
30+
}
31+
32+
function expectSkillsAdvertised(caps: Caps, where: string): void {
33+
expect(caps.extensions, `${where}: no extensions block`).toBeDefined();
34+
expect(
35+
caps.extensions?.[SKILLS_EXTENSION_ID],
36+
`${where}: skills extension not advertised`
37+
).toBeDefined();
38+
// resources/read is how every skill file is fetched, so a client that gates
39+
// on capabilities needs this too.
40+
expect(caps.resources, `${where}: resources not advertised`).toBeDefined();
41+
}
42+
43+
const SCENARIOS: Array<
44+
[string, () => SkillsNoPrefetchScenario | SkillsVerificationScenario]
45+
> = [
46+
['no-prefetch', () => new SkillsNoPrefetchScenario()],
47+
['verify-digest', () => new SkillsVerificationScenario('digest')],
48+
['verify-size', () => new SkillsVerificationScenario('size')],
49+
['verify-frontmatter', () => new SkillsVerificationScenario('frontmatter')]
50+
];
51+
52+
describe('SEP-2640 client scenarios advertise skills on both lifecycles', () => {
53+
test.each(SCENARIOS)('%s: server/discover', async (label, make) => {
54+
const scenario = make();
55+
const { serverUrl } = await scenario.start(testScenarioContext());
56+
try {
57+
const discover = await sendStatelessRequest(serverUrl, 'server/discover');
58+
expect(discover.status).toBe(200);
59+
expectSkillsAdvertised(
60+
capsOf(discover.body?.result),
61+
`${label} discover`
62+
);
63+
} finally {
64+
await scenario.stop();
65+
}
66+
});
67+
68+
test.each(SCENARIOS)('%s: initialize', async (label, make) => {
69+
const scenario = make();
70+
const { serverUrl } = await scenario.start(testScenarioContext());
71+
try {
72+
const init = await sendStatelessRequest(serverUrl, 'initialize', {
73+
protocolVersion: DRAFT_PROTOCOL_VERSION,
74+
capabilities: {},
75+
clientInfo: { name: 'capabilities-test', version: '1.0.0' }
76+
});
77+
expect(init.status).toBe(200);
78+
expectSkillsAdvertised(capsOf(init.body?.result), `${label} initialize`);
79+
} finally {
80+
await scenario.stop();
81+
}
82+
});
83+
});
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/**
2+
* Results for the hand-rolled SEP-2640 client-scenario mocks, shaped like a
3+
* conformant server's so a strict client is graded on its skills behaviour
4+
* rather than turned away by the harness.
5+
*/
6+
7+
import {
8+
DRAFT_PROTOCOL_VERSION,
9+
NEGOTIABLE_PROTOCOL_VERSIONS
10+
} from '../../../types.js';
11+
import { withRequiredDraftResultFields } from '../../../mock-server/index.js';
12+
13+
/**
14+
* `ListSkillsResult` is cacheable like `resources/list`, but `skills/list` is
15+
* an extension method and not in the shared cacheable set, so the caching
16+
* members are stamped here.
17+
*/
18+
export function skillsListResult(skills: object[]): object {
19+
return { resultType: 'complete', ttlMs: 0, cacheScope: 'private', skills };
20+
}
21+
22+
export function readResult(uri: unknown, text: string): unknown {
23+
return withRequiredDraftResultFields('resources/read', {
24+
contents: [{ uri, mimeType: 'text/markdown', text }]
25+
});
26+
}
27+
28+
/**
29+
* The `initialize` result at the version the client asked for, when the suite
30+
* can speak it. The extension has no 2026-07-28 dependency, and the runner
31+
* hands these scenarios' clients 2025-11-25 unless `--spec-version` says
32+
* otherwise, so a fixed 2026-07-28 reply would turn a correct stateful client
33+
* away before it ever lists.
34+
*/
35+
export function initializeResult(
36+
name: string,
37+
request: { params?: { protocolVersion?: unknown } },
38+
capabilities: object
39+
): object {
40+
const requested = request.params?.protocolVersion;
41+
const protocolVersion =
42+
typeof requested === 'string' &&
43+
NEGOTIABLE_PROTOCOL_VERSIONS.includes(requested)
44+
? requested
45+
: DRAFT_PROTOCOL_VERSION;
46+
return {
47+
resultType: 'complete',
48+
protocolVersion,
49+
serverInfo: { name: `${name}-server`, version: '1.0.0' },
50+
capabilities
51+
};
52+
}

0 commit comments

Comments
 (0)