From 3430be542aa091ad891fcd627e44a7582063e16d Mon Sep 17 00:00:00 2001 From: itelo Date: Fri, 21 Aug 2026 11:45:35 -0300 Subject: [PATCH 1/2] fix(harness): declare typebox so pi-mcp-adapter resolves it at runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pi-coding-agent pins typebox exactly, so npm nested it under the coding-agent and left no top-level copy. pi-mcp-adapter also imports typebox and resolves it from its own dir, so on any clean install it threw `Cannot find module 'typebox'` the moment the pi harness loaded the MCP adapter — the pi run failed instantly with an empty diff. CI missed it because the eval (the only runtime exercise of pi) doesn't run in CI. Declare typebox (pinned 1.1.38) as a direct dep so it hoists to top-level where pi-mcp-adapter can find it. Also make the eval surface failures instead of hiding them: the real runner captures the step-failure reason (runIntegration reports it as an event, not a throw), threads it onto the case result, and run.ts prints a Failures section. --- eval/real-runner.ts | 9 ++- eval/run-case.ts | 3 + eval/run.ts | 12 ++++ package-lock.json | 113 +++---------------------------------- package.json | 3 +- test/eval/run-case.test.ts | 19 +++++++ 6 files changed, 53 insertions(+), 106 deletions(-) diff --git a/eval/real-runner.ts b/eval/real-runner.ts index a13c2f1..5dadea9 100644 --- a/eval/real-runner.ts +++ b/eval/real-runner.ts @@ -18,6 +18,10 @@ export function createRealRunner(token: string): CaseRunner { let ok = false let costUsd: number | null = null + // The first step-failure reason — runIntegration catches per-step errors and + // reports them as events rather than throwing, so capture it here to surface + // why a case failed instead of leaving it a silent `no`. + let error: string | undefined await runIntegration({ root: workDir, sdk: context.config.sdk, @@ -28,12 +32,15 @@ export function createRealRunner(token: string): CaseRunner { mode: spec.mode, signal: context.signal, onEvent: (event) => { + if (event.kind === 'step_failed' && error == null) { + error = event.reason + } if (event.kind === 'done') { ok = event.ok costUsd = event.cost_usd } }, }) - return { ok, costUsd } + return { ok, costUsd, ...(error != null ? { error } : {}) } } } diff --git a/eval/run-case.ts b/eval/run-case.ts index 8eb5019..ca119fc 100644 --- a/eval/run-case.ts +++ b/eval/run-case.ts @@ -10,6 +10,8 @@ import { captureDiff, prepareFixture } from './workspace.js' export interface RunOutcome { ok: boolean costUsd: number | null + // Why the run failed, when the runner caught a reason rather than throwing. + error?: string } export interface RunContext { @@ -48,6 +50,7 @@ export async function runCase(args: { const outcome = await runner(workDir, spec, { config, signal }) ok = outcome.ok costUsd = outcome.costUsd + error = outcome.error } catch (caught) { error = caught instanceof Error ? caught.message : String(caught) } diff --git a/eval/run.ts b/eval/run.ts index 4e018ae..e1a4a16 100644 --- a/eval/run.ts +++ b/eval/run.ts @@ -65,6 +65,18 @@ async function main(): Promise { write('') write(formatReport(results)) + + // Surface why any case failed, so a `no` row is never a dead end. + const failures = results.filter((result) => result.error != null) + if (failures.length > 0) { + write('') + write('Failures:') + for (const failure of failures) { + write( + ` ${failure.fixture} · ${failure.mode} · ${failure.harness}: ${failure.error}`, + ) + } + } } function readFixtureConfig(fixture: string): FixtureConfig { diff --git a/package-lock.json b/package-lock.json index ec5d38a..c597883 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,8 @@ "jiti": "^2.7.0", "minimist": "^1.2.8", "open": "^11.0.0", - "pi-mcp-adapter": "~2.15.0" + "pi-mcp-adapter": "~2.15.0", + "typebox": "1.1.38" }, "devDependencies": { "@seamapi/cli": "^0.32.0", @@ -123,9 +124,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -139,9 +137,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -155,9 +150,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -171,9 +163,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -831,7 +820,7 @@ "typebox": "1.1.38" }, "bin": { - "pi-ai": "dist/cli.js" + "pi-ai": "./dist/cli.js" }, "engines": { "node": ">=22.19.0" @@ -947,9 +936,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -966,9 +952,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -985,9 +968,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1004,9 +984,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1023,9 +1000,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3132,9 +3106,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3151,9 +3122,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3170,9 +3138,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3189,9 +3154,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3208,9 +3170,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3460,9 +3419,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3480,9 +3436,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3500,9 +3453,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3520,9 +3470,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3540,9 +3487,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3560,9 +3504,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4285,9 +4226,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4302,9 +4240,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4319,9 +4254,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4336,9 +4268,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4353,9 +4282,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4370,9 +4296,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4387,9 +4310,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4404,9 +4324,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4421,9 +4338,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4438,9 +4352,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -8911,9 +8822,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8935,9 +8843,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8959,9 +8864,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8983,9 +8885,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -11674,6 +11573,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", diff --git a/package.json b/package.json index 8591cd3..a6f2b6c 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,8 @@ "jiti": "^2.7.0", "minimist": "^1.2.8", "open": "^11.0.0", - "pi-mcp-adapter": "~2.15.0" + "pi-mcp-adapter": "~2.15.0", + "typebox": "1.1.38" }, "devDependencies": { "@seamapi/cli": "^0.32.0", diff --git a/test/eval/run-case.test.ts b/test/eval/run-case.test.ts index b43d24e..be2b442 100644 --- a/test/eval/run-case.test.ts +++ b/test/eval/run-case.test.ts @@ -100,6 +100,25 @@ test('runCase: attaches the scorer result when a diff is produced', async () => expect(result.score?.total).toBe(0.75) }) +test('runCase: surfaces a runner-reported error (no throw)', async () => { + const fixtureDir = makeFixtureDir() + const result = await runCase({ + fixtureDir, + spec: { fixture: 'demo', mode: 'full_api', harness: 'pi' }, + config, + signal: new AbortController().signal, + now: fakeClock(), + runner: async () => ({ + ok: false, + costUsd: null, + error: "Cannot find module 'typebox'", + }), + }) + + expect(result.ok).toBe(false) + expect(result.error).toBe("Cannot find module 'typebox'") +}) + test('runCase: a thrown scorer never fails the case', async () => { const fixtureDir = makeFixtureDir() const result = await runCase({ From 0988cb7a6b8a6ea0f3dfe671cfe7596dd84a7501 Mon Sep 17 00:00:00 2001 From: Seam Bot Date: Fri, 21 Aug 2026 14:46:18 +0000 Subject: [PATCH 2/2] ci: Generate code --- package-lock.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index c597883..2f93a27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -820,7 +820,7 @@ "typebox": "1.1.38" }, "bin": { - "pi-ai": "./dist/cli.js" + "pi-ai": "dist/cli.js" }, "engines": { "node": ">=22.19.0" @@ -936,6 +936,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -952,6 +955,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -968,6 +974,9 @@ "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -984,6 +993,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1000,6 +1012,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [