diff --git a/eval/real-runner.ts b/eval/real-runner.ts index a13c2f1..5dadea9 100644 --- a/eval/real-runner.ts +++ b/eval/real-runner.ts @@ -18,6 +18,10 @@ export function createRealRunner(token: string): CaseRunner { let ok = false let costUsd: number | null = null + // The first step-failure reason — runIntegration catches per-step errors and + // reports them as events rather than throwing, so capture it here to surface + // why a case failed instead of leaving it a silent `no`. + let error: string | undefined await runIntegration({ root: workDir, sdk: context.config.sdk, @@ -28,12 +32,15 @@ export function createRealRunner(token: string): CaseRunner { mode: spec.mode, signal: context.signal, onEvent: (event) => { + if (event.kind === 'step_failed' && error == null) { + error = event.reason + } if (event.kind === 'done') { ok = event.ok costUsd = event.cost_usd } }, }) - return { ok, costUsd } + return { ok, costUsd, ...(error != null ? { error } : {}) } } } diff --git a/eval/run-case.ts b/eval/run-case.ts index 8eb5019..ca119fc 100644 --- a/eval/run-case.ts +++ b/eval/run-case.ts @@ -10,6 +10,8 @@ import { captureDiff, prepareFixture } from './workspace.js' export interface RunOutcome { ok: boolean costUsd: number | null + // Why the run failed, when the runner caught a reason rather than throwing. + error?: string } export interface RunContext { @@ -48,6 +50,7 @@ export async function runCase(args: { const outcome = await runner(workDir, spec, { config, signal }) ok = outcome.ok costUsd = outcome.costUsd + error = outcome.error } catch (caught) { error = caught instanceof Error ? caught.message : String(caught) } diff --git a/eval/run.ts b/eval/run.ts index 4e018ae..e1a4a16 100644 --- a/eval/run.ts +++ b/eval/run.ts @@ -65,6 +65,18 @@ async function main(): Promise { write('') write(formatReport(results)) + + // Surface why any case failed, so a `no` row is never a dead end. + const failures = results.filter((result) => result.error != null) + if (failures.length > 0) { + write('') + write('Failures:') + for (const failure of failures) { + write( + ` ${failure.fixture} · ${failure.mode} · ${failure.harness}: ${failure.error}`, + ) + } + } } function readFixtureConfig(fixture: string): FixtureConfig { diff --git a/package-lock.json b/package-lock.json index ec5d38a..2f93a27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,8 @@ "jiti": "^2.7.0", "minimist": "^1.2.8", "open": "^11.0.0", - "pi-mcp-adapter": "~2.15.0" + "pi-mcp-adapter": "~2.15.0", + "typebox": "1.1.38" }, "devDependencies": { "@seamapi/cli": "^0.32.0", @@ -123,9 +124,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -139,9 +137,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -155,9 +150,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -171,9 +163,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -3132,9 +3121,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3151,9 +3137,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3170,9 +3153,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3189,9 +3169,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3208,9 +3185,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3460,9 +3434,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3480,9 +3451,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3500,9 +3468,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3520,9 +3485,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3540,9 +3502,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3560,9 +3519,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4285,9 +4241,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4302,9 +4255,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4319,9 +4269,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4336,9 +4283,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4353,9 +4297,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4370,9 +4311,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4387,9 +4325,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4404,9 +4339,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4421,9 +4353,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4438,9 +4367,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -8911,9 +8837,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8935,9 +8858,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8959,9 +8879,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8983,9 +8900,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -11674,6 +11588,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", diff --git a/package.json b/package.json index 8591cd3..a6f2b6c 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,8 @@ "jiti": "^2.7.0", "minimist": "^1.2.8", "open": "^11.0.0", - "pi-mcp-adapter": "~2.15.0" + "pi-mcp-adapter": "~2.15.0", + "typebox": "1.1.38" }, "devDependencies": { "@seamapi/cli": "^0.32.0", diff --git a/test/eval/run-case.test.ts b/test/eval/run-case.test.ts index b43d24e..be2b442 100644 --- a/test/eval/run-case.test.ts +++ b/test/eval/run-case.test.ts @@ -100,6 +100,25 @@ test('runCase: attaches the scorer result when a diff is produced', async () => expect(result.score?.total).toBe(0.75) }) +test('runCase: surfaces a runner-reported error (no throw)', async () => { + const fixtureDir = makeFixtureDir() + const result = await runCase({ + fixtureDir, + spec: { fixture: 'demo', mode: 'full_api', harness: 'pi' }, + config, + signal: new AbortController().signal, + now: fakeClock(), + runner: async () => ({ + ok: false, + costUsd: null, + error: "Cannot find module 'typebox'", + }), + }) + + expect(result.ok).toBe(false) + expect(result.error).toBe("Cannot find module 'typebox'") +}) + test('runCase: a thrown scorer never fails the case', async () => { const fixtureDir = makeFixtureDir() const result = await runCase({