fix(tasks): report cancelled fixture status consistently

src/scenarios/server/tasks/lifecycle.test.ts

@@ -1,82 +1,220 @@
-import { describe, test, expect, afterEach } from 'vitest';
-import { testContext } from '../../../connection/testing';
-import { TasksLifecycleScenario } from './lifecycle';
-import { DRAFT_PROTOCOL_VERSION } from '../../../types';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import { createServerStateless, type MockServer } from '../../../mock-server';
+import { runServerConformanceTest } from '../../../runner/server';
 import type { ConformanceCheck } from '../../../types';
 
-/**
- * Pins the untestable-failure policy (issue #248) for the SEP-2663 lifecycle
- * scenario: when no task is ever created, the downstream task checks must
- * fail with a "Not testable:" cause instead of reporting SKIPPED.
- */
+type CancellationBehavior =
+  | 'cancelled'
+  | 'completed'
+  | 'failed'
+  | 'working'
+  | 'taskless';
 
-const realFetch = global.fetch;
-afterEach(() => {
-  global.fetch = realFetch;
-});
+async function startServer(
+  cancellation: CancellationBehavior
+): Promise<MockServer> {
+  const tasks = new Map<
+    string,
+    {
+      name: string;
+      createdAt: string;
+      completesAt: number;
+      status: 'working' | 'cancelled' | 'completed' | 'failed';
+    }
+  >();
 
-function mockServer() {
-  global.fetch = (async (_url: any, init: any) => {
-    const body = JSON.parse(init.body);
-    let result: any;
-    if (body.method === 'server/discover') {
-      result = {
-        supportedVersions: [DRAFT_PROTOCOL_VERSION],
-        capabilities: { tools: {} },
-        serverInfo: { name: 'taskless-server', version: '1.0.0' }
+  return createServerStateless({
+    'tools/call': (params) => {
+      if (params.name === 'greet' || cancellation === 'taskless') {
+        return { content: [{ type: 'text', text: 'Hello, World!' }] };
+      }
+      if (typeof params.name !== 'string') {
+        throw new Error('Expected a tool name');
+      }
+      const args = params.arguments as Record<string, unknown> | undefined;
+      const taskId = typeof args?.label === 'string' ? args.label : params.name;
+      const seconds = typeof args?.seconds === 'number' ? args.seconds : 1;
+      const task = {
+        name: params.name,
+        createdAt: new Date().toISOString(),
+        completesAt: Date.now() + seconds * 1000,
+        status: 'working' as const
       };
-    } else if (body.method === 'tools/list') {
-      result = { tools: [] };
-    } else if (body.method === 'tools/call') {
-      // Always answers synchronously: never creates a task.
-      result = {
-        resultType: 'complete',
-        content: [{ type: 'text', text: 'sync answer' }]
+      tasks.set(taskId, task);
+      return {
+        resultType: 'task',
+        taskId,
+        status: task.status,
+        createdAt: task.createdAt,
+        lastUpdatedAt: task.createdAt,
+        ttlMs: 60_000
       };
-    } else {
+    },
+    'tasks/cancel': (params) => {
+      const task = tasks.get(String(params.taskId));
+      if (!task) throw new Error('Unknown task');
+      if (task.status === 'working' && cancellation !== 'taskless') {
+        task.status = cancellation;
+      }
+      return { resultType: 'complete' };
+    },
+    'tasks/get': (params) => {
+      const task = tasks.get(String(params.taskId));
+      if (!task) throw new Error('Unknown task');
+      if (task.status === 'working' && Date.now() >= task.completesAt) {
+        task.status =
+          task.name === 'protocol_error_job' ? 'failed' : 'completed';
+      }
       return {
-        status: 404,
-        headers: { get: () => 'application/json' },
-        json: async () => ({
-          jsonrpc: '2.0',
-          id: body.id ?? null,
-          error: { code: -32601, message: 'Method not found' }
-        }),
-        text: async () =>
-          JSON.stringify({
-            jsonrpc: '2.0',
-            id: body.id ?? null,
-            error: { code: -32601, message: 'Method not found' }
-          })
-      } as unknown as Response;
+        taskId: params.taskId,
+        status: task.status,
+        createdAt: task.createdAt,
+        lastUpdatedAt: new Date().toISOString(),
+        ttlMs: 60_000,
+        ...(task.status === 'completed'
+          ? {
+              result: {
+                content: [{ type: 'text', text: 'done' }],
+                isError: task.name === 'failing_job'
+              }
+            }
+          : {}),
+        ...(task.status === 'failed'
+          ? { error: { code: -32603, message: 'Internal error' } }
+          : {})
+      };
     }
-    const payload = { jsonrpc: '2.0', id: body.id, result };
-    return {
-      status: 200,
-      headers: { get: () => 'application/json' },
-      json: async () => payload,
-      text: async () => JSON.stringify(payload)
-    } as unknown as Response;
-  }) as typeof fetch;
-  return 'http://mock-taskless-server.local';
+  });
 }
 
-describe('tasks-lifecycle — no task created', () => {
-  test('downstream task checks fail as untestable instead of SKIPPED', async () => {
-    const mockUrl = mockServer();
-    const scenario = new TasksLifecycleScenario();
-    const checks: ConformanceCheck[] = await scenario.run(
-      testContext(mockUrl, DRAFT_PROTOCOL_VERSION)
+describe('tasks-lifecycle cancellation reports', () => {
+  let server: MockServer | undefined;
+  let outputDir: string;
+
+  beforeEach(async () => {
+    outputDir = await mkdtemp(path.join(tmpdir(), 'tasks-lifecycle-'));
+  });
+
+  afterEach(async () => {
+    await server?.close();
+    await rm(outputDir, { recursive: true, force: true });
+  });
+
+  async function runLifecycle(
+    cancellation: CancellationBehavior
+  ): Promise<ConformanceCheck[]> {
+    server = await startServer(cancellation);
+    const result = await runServerConformanceTest(
+      server.url,
+      'tasks-lifecycle',
+      outputDir
     );
+    expect(result.resultDir).toBeDefined();
+    const checks: ConformanceCheck[] = JSON.parse(
+      await readFile(path.join(result.resultDir!, 'checks.json'), 'utf8')
+    );
+    expect(checks).toEqual(JSON.parse(JSON.stringify(result.checks)));
+    expect(checks.filter((check) => check.id === 'wire-schema-valid')).toEqual([
+      expect.objectContaining({ status: 'SUCCESS' })
+    ]);
+    return checks;
+  }
+
+  test.each(['cancelled', 'completed', 'failed'] as const)(
+    'reports the fixture contract when cancellation settles to %s',
+    async (status) => {
+      const checks = await runLifecycle(status);
+      expect(
+        checks.find((check) => check.id === 'sep-2663-cancel-ack-empty-result')
+      ).toMatchObject({
+        status: 'SUCCESS',
+        details: { statusAfterCancel: status }
+      });
+      expect(
+        checks.filter(
+          (check) => check.id === 'sep-2663-tasks-get-status-cancelled'
+        )
+      ).toEqual([
+        expect.objectContaining({
+          status: status === 'cancelled' ? 'SUCCESS' : 'FAILURE',
+          ...(status === 'cancelled'
+            ? {}
+            : {
+                errorMessage:
+                  `slow_compute fixture contract requires status:"cancelled" ` +
+                  `after cancellation while running; got "${status}"`
+              }),
+          details: { statusAfterCancel: status }
+        })
+      ]);
+      expect(checks.filter((check) => check.status !== 'SUCCESS')).toHaveLength(
+        status === 'cancelled' ? 0 : 1
+      );
+    },
+    20_000
+  );
 
-    const gated = checks.filter((c) =>
-      c.errorMessage?.startsWith('Not testable:')
+  test('reports cancelled status as untestable when no task is created', async () => {
+    const checks = await runLifecycle('taskless');
+
+    expect(
+      checks.filter(
+        (check) => check.id === 'sep-2663-tasks-get-status-cancelled'
+      )
+    ).toEqual([
+      expect.objectContaining({
+        status: 'FAILURE',
+        errorMessage: expect.stringMatching(
+          /^Not testable:.*did not create a task/
+        ),
+        details: expect.objectContaining({ untestable: true })
+      })
+    ]);
+    const gated = checks.filter((check) =>
+      check.errorMessage?.startsWith('Not testable:')
     );
-    expect(gated.length).toBeGreaterThan(0);
     for (const check of gated) {
       expect(check.status).toBe('FAILURE');
       expect(check.details).toMatchObject({ untestable: true });
     }
-    expect(checks.every((c) => c.status !== 'SKIPPED')).toBe(true);
+    expect(checks.every((check) => check.status !== 'SKIPPED')).toBe(true);
+    expect(
+      server?.recorded.some((request) => request.method === 'tasks/cancel')
+    ).toBe(false);
   });
+
+  test('reports cancelled status as untestable when terminal polling times out', async () => {
+    const checks = await runLifecycle('working');
+
+    expect(
+      checks.find((check) => check.id === 'sep-2663-cancel-ack-empty-result')
+    ).toMatchObject({
+      status: 'FAILURE',
+      errorMessage:
+        'Task lifecycle-cancel did not reach terminal state within 10000ms'
+    });
+    expect(
+      checks.filter(
+        (check) => check.id === 'sep-2663-tasks-get-status-cancelled'
+      )
+    ).toEqual([
+      expect.objectContaining({
+        status: 'FAILURE',
+        errorMessage: expect.stringMatching(
+          /^Not testable:.*Task lifecycle-cancel did not reach terminal state within 10000ms/
+        ),
+        details: expect.objectContaining({ untestable: true })
+      })
+    ]);
+    expect(
+      server?.recorded.filter(
+        (request) =>
+          request.method === 'tasks/get' &&
+          request.params?.taskId === 'lifecycle-cancel'
+      ).length
+    ).toBeGreaterThan(1);
+  }, 25_000);
 });

src/scenarios/server/tasks/lifecycle.ts

@@ -423,7 +423,11 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under
       const id = 'sep-2663-cancel-ack-empty-result';
       const name = 'TasksCancelEmptyAck';
       const description =
-        'tasks/cancel returns {resultType:"complete"} ack; status settles to cancelled';
+        'tasks/cancel returns an empty {resultType:"complete"} acknowledgement';
+      const cancelledId = 'sep-2663-tasks-get-status-cancelled';
+      const cancelledName = 'TasksGetCancelledStatus';
+      const cancelledDescription =
+        'The cancellable slow_compute fixture settles to cancelled after tasks/cancel';
       let cancelTaskId: string | undefined;
       try {
         const created = (await conn.request('tools/call', {
@@ -441,6 +445,15 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under
             errorMessage: 'slow_compute did not create a task',
             specReferences: [SEP_2663_REF, SEP_2322_REF]
           });
+          checks.push(
+            untestableCheck(
+              cancelledId,
+              cancelledName,
+              cancelledDescription,
+              'slow_compute did not create a task, so its cancelled status could not be observed',
+              [SEP_2663_REF]
+            )
+          );
         } else {
           const ack = (await conn.request('tasks/cancel', {
             taskId: cancelTaskId
@@ -461,8 +474,6 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under
               `cancel ack MUST NOT carry task-envelope fields; got: ${ackOffenders.join(', ')}`
             );
           }
-          // SEP-2663 §Task Cancellation: transition to `cancelled` is not
-          // guaranteed; record the settled status as diagnostic detail only.
           const after = await waitForTerminal(conn, cancelTaskId);
           checks.push({
             id,
@@ -474,9 +485,31 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under
             specReferences: [SEP_2663_REF, SEP_2322_REF],
             details: { cancelAck: ack, statusAfterCancel: after.status }
           });
+          checks.push({
+            id: cancelledId,
+            name: cancelledName,
+            description: cancelledDescription,
+            status: after.status === 'cancelled' ? 'SUCCESS' : 'FAILURE',
+            timestamp: new Date().toISOString(),
+            errorMessage:
+              after.status === 'cancelled'
+                ? undefined
+                : `slow_compute fixture contract requires status:"cancelled" after cancellation while running; got ${JSON.stringify(after.status)}`,
+            specReferences: [SEP_2663_REF],
+            details: { statusAfterCancel: after.status }
+          });
         }
       } catch (error) {
         checks.push(failureCheck(id, name, description, error, [SEP_2663_REF]));
+        checks.push(
+          untestableCheck(
+            cancelledId,
+            cancelledName,
+            cancelledDescription,
+            `could not complete the slow_compute cancellation probe: ${errMsg(error)}`,
+            [SEP_2663_REF]
+          )
+        );
       }
     }