Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion eval/real-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ export function createRealRunner(token: string): CaseRunner {

let ok = false
let costUsd: number | null = null
// The first step-failure reason — runIntegration catches per-step errors and
// reports them as events rather than throwing, so capture it here to surface
// why a case failed instead of leaving it a silent `no`.
let error: string | undefined
await runIntegration({
root: workDir,
sdk: context.config.sdk,
Expand All @@ -28,12 +32,15 @@ export function createRealRunner(token: string): CaseRunner {
mode: spec.mode,
signal: context.signal,
onEvent: (event) => {
if (event.kind === 'step_failed' && error == null) {
error = event.reason
}
if (event.kind === 'done') {
ok = event.ok
costUsd = event.cost_usd
}
},
})
return { ok, costUsd }
return { ok, costUsd, ...(error != null ? { error } : {}) }
}
}
3 changes: 3 additions & 0 deletions eval/run-case.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import { captureDiff, prepareFixture } from './workspace.js'
export interface RunOutcome {
ok: boolean
costUsd: number | null
// Why the run failed, when the runner caught a reason rather than throwing.
error?: string
}

export interface RunContext {
Expand Down Expand Up @@ -48,6 +50,7 @@ export async function runCase(args: {
const outcome = await runner(workDir, spec, { config, signal })
ok = outcome.ok
costUsd = outcome.costUsd
error = outcome.error
} catch (caught) {
error = caught instanceof Error ? caught.message : String(caught)
}
Expand Down
12 changes: 12 additions & 0 deletions eval/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,18 @@ async function main(): Promise<void> {

write('')
write(formatReport(results))

// Surface why any case failed, so a `no` row is never a dead end.
const failures = results.filter((result) => result.error != null)
if (failures.length > 0) {
write('')
write('Failures:')
for (const failure of failures) {
write(
` ${failure.fixture} · ${failure.mode} · ${failure.harness}: ${failure.error}`,
)
}
}
}

function readFixtureConfig(fixture: string): FixtureConfig {
Expand Down
96 changes: 8 additions & 88 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@
"jiti": "^2.7.0",
"minimist": "^1.2.8",
"open": "^11.0.0",
"pi-mcp-adapter": "~2.15.0"
"pi-mcp-adapter": "~2.15.0",
"typebox": "1.1.38"
},
"devDependencies": {
"@seamapi/cli": "^0.32.0",
Expand Down
Loading
Loading