diff --git a/src/scenarios/server/tasks/lifecycle.test.ts b/src/scenarios/server/tasks/lifecycle.test.ts index 8926538b..de2dd021 100644 --- a/src/scenarios/server/tasks/lifecycle.test.ts +++ b/src/scenarios/server/tasks/lifecycle.test.ts @@ -1,82 +1,220 @@ -import { describe, test, expect, afterEach } from 'vitest'; -import { testContext } from '../../../connection/testing'; -import { TasksLifecycleScenario } from './lifecycle'; -import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { createServerStateless, type MockServer } from '../../../mock-server'; +import { runServerConformanceTest } from '../../../runner/server'; import type { ConformanceCheck } from '../../../types'; -/** - * Pins the untestable-failure policy (issue #248) for the SEP-2663 lifecycle - * scenario: when no task is ever created, the downstream task checks must - * fail with a "Not testable:" cause instead of reporting SKIPPED. - */ +type CancellationBehavior = + | 'cancelled' + | 'completed' + | 'failed' + | 'working' + | 'taskless'; -const realFetch = global.fetch; -afterEach(() => { - global.fetch = realFetch; -}); +async function startServer( + cancellation: CancellationBehavior +): Promise { + const tasks = new Map< + string, + { + name: string; + createdAt: string; + completesAt: number; + status: 'working' | 'cancelled' | 'completed' | 'failed'; + } + >(); -function mockServer() { - global.fetch = (async (_url: any, init: any) => { - const body = JSON.parse(init.body); - let result: any; - if (body.method === 'server/discover') { - result = { - supportedVersions: [DRAFT_PROTOCOL_VERSION], - capabilities: { tools: {} }, - serverInfo: { name: 'taskless-server', version: '1.0.0' } + return createServerStateless({ + 'tools/call': (params) => { + if (params.name === 'greet' || cancellation === 'taskless') { + return { content: [{ type: 'text', text: 'Hello, World!' }] }; + } + if (typeof params.name !== 'string') { + throw new Error('Expected a tool name'); + } + const args = params.arguments as Record | undefined; + const taskId = typeof args?.label === 'string' ? args.label : params.name; + const seconds = typeof args?.seconds === 'number' ? args.seconds : 1; + const task = { + name: params.name, + createdAt: new Date().toISOString(), + completesAt: Date.now() + seconds * 1000, + status: 'working' as const }; - } else if (body.method === 'tools/list') { - result = { tools: [] }; - } else if (body.method === 'tools/call') { - // Always answers synchronously: never creates a task. - result = { - resultType: 'complete', - content: [{ type: 'text', text: 'sync answer' }] + tasks.set(taskId, task); + return { + resultType: 'task', + taskId, + status: task.status, + createdAt: task.createdAt, + lastUpdatedAt: task.createdAt, + ttlMs: 60_000 }; - } else { + }, + 'tasks/cancel': (params) => { + const task = tasks.get(String(params.taskId)); + if (!task) throw new Error('Unknown task'); + if (task.status === 'working' && cancellation !== 'taskless') { + task.status = cancellation; + } + return { resultType: 'complete' }; + }, + 'tasks/get': (params) => { + const task = tasks.get(String(params.taskId)); + if (!task) throw new Error('Unknown task'); + if (task.status === 'working' && Date.now() >= task.completesAt) { + task.status = + task.name === 'protocol_error_job' ? 'failed' : 'completed'; + } return { - status: 404, - headers: { get: () => 'application/json' }, - json: async () => ({ - jsonrpc: '2.0', - id: body.id ?? null, - error: { code: -32601, message: 'Method not found' } - }), - text: async () => - JSON.stringify({ - jsonrpc: '2.0', - id: body.id ?? null, - error: { code: -32601, message: 'Method not found' } - }) - } as unknown as Response; + taskId: params.taskId, + status: task.status, + createdAt: task.createdAt, + lastUpdatedAt: new Date().toISOString(), + ttlMs: 60_000, + ...(task.status === 'completed' + ? { + result: { + content: [{ type: 'text', text: 'done' }], + isError: task.name === 'failing_job' + } + } + : {}), + ...(task.status === 'failed' + ? { error: { code: -32603, message: 'Internal error' } } + : {}) + }; } - const payload = { jsonrpc: '2.0', id: body.id, result }; - return { - status: 200, - headers: { get: () => 'application/json' }, - json: async () => payload, - text: async () => JSON.stringify(payload) - } as unknown as Response; - }) as typeof fetch; - return 'http://mock-taskless-server.local'; + }); } -describe('tasks-lifecycle — no task created', () => { - test('downstream task checks fail as untestable instead of SKIPPED', async () => { - const mockUrl = mockServer(); - const scenario = new TasksLifecycleScenario(); - const checks: ConformanceCheck[] = await scenario.run( - testContext(mockUrl, DRAFT_PROTOCOL_VERSION) +describe('tasks-lifecycle cancellation reports', () => { + let server: MockServer | undefined; + let outputDir: string; + + beforeEach(async () => { + outputDir = await mkdtemp(path.join(tmpdir(), 'tasks-lifecycle-')); + }); + + afterEach(async () => { + await server?.close(); + await rm(outputDir, { recursive: true, force: true }); + }); + + async function runLifecycle( + cancellation: CancellationBehavior + ): Promise { + server = await startServer(cancellation); + const result = await runServerConformanceTest( + server.url, + 'tasks-lifecycle', + outputDir ); + expect(result.resultDir).toBeDefined(); + const checks: ConformanceCheck[] = JSON.parse( + await readFile(path.join(result.resultDir!, 'checks.json'), 'utf8') + ); + expect(checks).toEqual(JSON.parse(JSON.stringify(result.checks))); + expect(checks.filter((check) => check.id === 'wire-schema-valid')).toEqual([ + expect.objectContaining({ status: 'SUCCESS' }) + ]); + return checks; + } + + test.each(['cancelled', 'completed', 'failed'] as const)( + 'reports the fixture contract when cancellation settles to %s', + async (status) => { + const checks = await runLifecycle(status); + expect( + checks.find((check) => check.id === 'sep-2663-cancel-ack-empty-result') + ).toMatchObject({ + status: 'SUCCESS', + details: { statusAfterCancel: status } + }); + expect( + checks.filter( + (check) => check.id === 'sep-2663-tasks-get-status-cancelled' + ) + ).toEqual([ + expect.objectContaining({ + status: status === 'cancelled' ? 'SUCCESS' : 'FAILURE', + ...(status === 'cancelled' + ? {} + : { + errorMessage: + `slow_compute fixture contract requires status:"cancelled" ` + + `after cancellation while running; got "${status}"` + }), + details: { statusAfterCancel: status } + }) + ]); + expect(checks.filter((check) => check.status !== 'SUCCESS')).toHaveLength( + status === 'cancelled' ? 0 : 1 + ); + }, + 20_000 + ); - const gated = checks.filter((c) => - c.errorMessage?.startsWith('Not testable:') + test('reports cancelled status as untestable when no task is created', async () => { + const checks = await runLifecycle('taskless'); + + expect( + checks.filter( + (check) => check.id === 'sep-2663-tasks-get-status-cancelled' + ) + ).toEqual([ + expect.objectContaining({ + status: 'FAILURE', + errorMessage: expect.stringMatching( + /^Not testable:.*did not create a task/ + ), + details: expect.objectContaining({ untestable: true }) + }) + ]); + const gated = checks.filter((check) => + check.errorMessage?.startsWith('Not testable:') ); - expect(gated.length).toBeGreaterThan(0); for (const check of gated) { expect(check.status).toBe('FAILURE'); expect(check.details).toMatchObject({ untestable: true }); } - expect(checks.every((c) => c.status !== 'SKIPPED')).toBe(true); + expect(checks.every((check) => check.status !== 'SKIPPED')).toBe(true); + expect( + server?.recorded.some((request) => request.method === 'tasks/cancel') + ).toBe(false); }); + + test('reports cancelled status as untestable when terminal polling times out', async () => { + const checks = await runLifecycle('working'); + + expect( + checks.find((check) => check.id === 'sep-2663-cancel-ack-empty-result') + ).toMatchObject({ + status: 'FAILURE', + errorMessage: + 'Task lifecycle-cancel did not reach terminal state within 10000ms' + }); + expect( + checks.filter( + (check) => check.id === 'sep-2663-tasks-get-status-cancelled' + ) + ).toEqual([ + expect.objectContaining({ + status: 'FAILURE', + errorMessage: expect.stringMatching( + /^Not testable:.*Task lifecycle-cancel did not reach terminal state within 10000ms/ + ), + details: expect.objectContaining({ untestable: true }) + }) + ]); + expect( + server?.recorded.filter( + (request) => + request.method === 'tasks/get' && + request.params?.taskId === 'lifecycle-cancel' + ).length + ).toBeGreaterThan(1); + }, 25_000); }); diff --git a/src/scenarios/server/tasks/lifecycle.ts b/src/scenarios/server/tasks/lifecycle.ts index 2214ff16..819dd15b 100644 --- a/src/scenarios/server/tasks/lifecycle.ts +++ b/src/scenarios/server/tasks/lifecycle.ts @@ -423,7 +423,11 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under const id = 'sep-2663-cancel-ack-empty-result'; const name = 'TasksCancelEmptyAck'; const description = - 'tasks/cancel returns {resultType:"complete"} ack; status settles to cancelled'; + 'tasks/cancel returns an empty {resultType:"complete"} acknowledgement'; + const cancelledId = 'sep-2663-tasks-get-status-cancelled'; + const cancelledName = 'TasksGetCancelledStatus'; + const cancelledDescription = + 'The cancellable slow_compute fixture settles to cancelled after tasks/cancel'; let cancelTaskId: string | undefined; try { const created = (await conn.request('tools/call', { @@ -441,6 +445,15 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under errorMessage: 'slow_compute did not create a task', specReferences: [SEP_2663_REF, SEP_2322_REF] }); + checks.push( + untestableCheck( + cancelledId, + cancelledName, + cancelledDescription, + 'slow_compute did not create a task, so its cancelled status could not be observed', + [SEP_2663_REF] + ) + ); } else { const ack = (await conn.request('tasks/cancel', { taskId: cancelTaskId @@ -461,8 +474,6 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under `cancel ack MUST NOT carry task-envelope fields; got: ${ackOffenders.join(', ')}` ); } - // SEP-2663 §Task Cancellation: transition to `cancelled` is not - // guaranteed; record the settled status as diagnostic detail only. const after = await waitForTerminal(conn, cancelTaskId); checks.push({ id, @@ -474,9 +485,31 @@ The server MUST advertise \`io.modelcontextprotocol/tasks\` under specReferences: [SEP_2663_REF, SEP_2322_REF], details: { cancelAck: ack, statusAfterCancel: after.status } }); + checks.push({ + id: cancelledId, + name: cancelledName, + description: cancelledDescription, + status: after.status === 'cancelled' ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: + after.status === 'cancelled' + ? undefined + : `slow_compute fixture contract requires status:"cancelled" after cancellation while running; got ${JSON.stringify(after.status)}`, + specReferences: [SEP_2663_REF], + details: { statusAfterCancel: after.status } + }); } } catch (error) { checks.push(failureCheck(id, name, description, error, [SEP_2663_REF])); + checks.push( + untestableCheck( + cancelledId, + cancelledName, + cancelledDescription, + `could not complete the slow_compute cancellation probe: ${errMsg(error)}`, + [SEP_2663_REF] + ) + ); } }