From 545ea4831b45912ae61342d692cacdcd074211bc Mon Sep 17 00:00:00 2001 From: Josh Owens Date: Wed, 30 Sep 2026 18:36:50 +0000 Subject: [PATCH 1/5] feat: grader file format and a no-artifact grade outcome A grader file is a TS/JS file whose default export is grade(artifactPath, { name: ({ result }) => { ... } }). A scenario picks one grader with "grader": { "file": "./grade.eval.ts", "name": "..." }. result offers assert(condition, message, context?), json(), text, path, shouldHave, shouldNotHave, and shouldMatch. Every check is collected instead of stopping at the first failure, and each failure prints its message with its context or with what the artifact held instead. A file grader compiles to a command grader: the engine runs `promptdiff grade --file --name ` in the sandbox, so the artifact path resolves against the sandbox like any command grader's paths. The new `grade` subcommand exits 0 (pass), 1 (failed checks), 77 (no artifact), or 2 (unusable file or name). Scenario loading checks the file and name in a child process before any paid run, and the baseline cache key includes the grader file's content. No artifact is now its own outcome. Command graders signal it by exiting 77 (also exported as $PROMPTDIFF_NO_ARTIFACT_EXIT_CODE) unless their expectExitCode is 77. Such runs are neither passes nor failures: pass rates cover graded runs only, summaries print "4/4 pass (100%), 1 no-artifact", compare notes each arm's count and fails an assertion for an arm with no graded run, and reports and receipts carry the count. The package now exports grade and its types from src/index.ts. Under `promptdiff grade`, "@theaiteam/promptdiff" and "promptdiff" resolve to the running copy, so grader files work without a local install. Refs #32 Co-Authored-By: Claude Opus 5.5 --- package.json | 4 + src/cli.ts | 68 +++++++++++- src/engine/cache.ts | 4 + src/engine/compare.ts | 68 ++++++++++-- src/engine/config.ts | 21 +++- src/engine/grader-file.ts | 151 ++++++++++++++++++++++++++ src/engine/grader.ts | 45 +++++++- src/engine/receipt.ts | 13 ++- src/engine/report.ts | 2 + src/grade.ts | 215 ++++++++++++++++++++++++++++++++++++++ src/index.ts | 10 ++ tsconfig.json | 2 +- 12 files changed, 584 insertions(+), 19 deletions(-) create mode 100644 src/engine/grader-file.ts create mode 100644 src/grade.ts create mode 100644 src/index.ts diff --git a/package.json b/package.json index 51d9fdb..9bd7a17 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,10 @@ "bin": { "promptdiff": "./promptdiff" }, + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, "files": [ "promptdiff", "src", diff --git a/src/cli.ts b/src/cli.ts index b1f5966..4b3588f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,6 +9,7 @@ import { renderStrict, resolveRenderVars, type RenderVars } from "./engine/rende import { appendNdjsonReport } from "./engine/report"; import { buildCompareReceipts, buildMeasureReceipts, writeReceipts } from "./engine/receipt"; import { formatCalibrationReport, runCalibration } from "./engine/judge"; +import { loadGraderFile, runGraderFile } from "./engine/grader-file"; import { assembleSystemPrompt } from "./prompt"; import { createRunner, RUNNER_NAMES, type RunnerName } from "./runner"; import type { ModelPricing } from "./runner/openai-compat"; @@ -115,6 +116,12 @@ export async function main(argv: string[]): Promise { } await cmdCalibrate(rest); return 0; + case "grade": + if (isHelp(rest)) { + console.log(gradeUsage()); + return 0; + } + return await cmdGrade(rest); default: throw new CliError(generalUsage()); } @@ -324,6 +331,30 @@ async function cmdCalibrate(argv: string[]): Promise { console.log(formatCalibrationReport(result)); } +const gradeSpecs: FlagSpecs = { + file: { arity: "one" }, + name: { arity: "one" }, + list: { arity: "none" }, +}; + +async function cmdGrade(argv: string[]): Promise { + const args = parseArgs(argv, gradeSpecs); + const file = args.one("file"); + const name = args.one("name"); + if (!file || (name === undefined) === !args.has("list")) { + throw new CliError(gradeUsage()); + } + if (args.has("list")) { + // Machine-readable: the scenario loader validates grader names with this. + console.log(JSON.stringify(Object.keys((await loadGraderFile(file)).graders))); + return 0; + } + const run = await runGraderFile(file, name ?? ""); + for (const line of run.stdout) console.log(line); + for (const line of run.stderr) console.error(line); + return run.exitCode; +} + async function cmdCompare(argv: string[]): Promise { const args = parseArgs(argv, compareSpecs); const scenario = args.one("scenario"); @@ -610,7 +641,10 @@ function compareUsage(): string { " (ops: == != > >= < <= contains; [*] passes if ANY element matches;", " works with every runner)", " command runs a shell command inside the per-run sandbox and checks exit code", - " (needs a tool-capable runner: claude-p)", + " (needs a tool-capable runner: claude-p); exit 77 means no artifact", + " file { \"file\": \"./grade.eval.ts\", \"name\": } runs one named grader", + " from a grader file (default export grade(artifactPath, { ... })) as a", + " command in the sandbox; see `promptdiff grade --help`", " judge an explicit judge model grades the final output against a markdown", " rubric: { \"type\": \"judge\", \"rubric\": \"./rubrics/r.md\", \"model\": ,", " \"runner\": , \"baseUrl\"?, \"minAccuracy\"? (default 0.9) }.", @@ -637,6 +671,12 @@ function compareUsage(): string { " used as literals. Scenarios may add their own \"render\" (scenario wins per", " var). Unbound placeholders fail before any paid run. Inline delivery only.", "", + "no artifact:", + " a run whose grader reports no artifact (exit 77) is neither a pass nor a", + " failure: pass rates cover graded runs only and the summary line counts it", + " (\"1/1 pass (100%), 1 no-artifact\"). An arm with no graded run fails its", + " target or regression assertion.", + "", "assertions:", " target baseline must not fully pass; proposed must beat baseline pass rate", " regression proposed must not fall below baseline pass rate", @@ -690,6 +730,9 @@ function measureUsage(): string { "--transcript-out additionally captures the per-event stream-json", "NDJSON as _measure_.stream.jsonl (claude-p only, large).", "", + "Runs whose grader reports no artifact (exit 77) are counted apart from", + "passes and failures: \"4/4 pass (100%), 1 no-artifact\".", + "", "Exit code is 0 whenever the runs complete — a measurement has no pass/fail.", ].join("\n"); } @@ -724,15 +767,36 @@ function calibrateUsage(): string { ].join("\n"); } +function gradeUsage(): string { + return [ + "usage: promptdiff grade --file (--name | --list)", + "", + "Runs one named grader from a grader file against its artifact, resolved", + "relative to the current directory. compare and measure run this same", + "command inside each run's sandbox; run it by hand inside a kept sandbox", + "(--keep-sandbox) to debug a grader without paying for new runs.", + "", + "exit codes:", + " 0 every check passed", + " 1 at least one check failed (all failures are printed to stderr)", + " 77 no artifact: the file does not exist. compare and measure count", + " this as no signal, not as a failure", + " 2 the grader file or name is unusable", + "", + "--list prints the file's grader names as a JSON array.", + ].join("\n"); +} + function generalUsage(): string { return [ - "usage: promptdiff [flags]", + "usage: promptdiff [flags]", "", "commands:", " run one bounded model invocation with inlined skills", " compare N-run baseline-vs-proposed scenario comparison", " measure N-run single-arm characterization (pass rates, no assertions)", " calibrate measure a judge grader against labeled rubric fixtures", + " grade run one named grader from a grader file (what file graders execute)", "", "use `promptdiff --help` for command flags", ].join("\n"); diff --git a/src/engine/cache.ts b/src/engine/cache.ts index 7378483..1e7af59 100644 --- a/src/engine/cache.ts +++ b/src/engine/cache.ts @@ -62,6 +62,10 @@ export function buildCacheKey(input: CacheKeyInput): string { mode: input.mode, delivery: input.delivery, grader: input.grader, + // A grader file's path is in the spec but its checks are in its content; + // editing the checks must miss. Undefined for other graders, so their + // keys are unchanged. + graderFile: input.grader.type === "file" ? sha256(readFileSync(input.grader.file)) : undefined, images: input.images.map((image) => sha256(readFileSync(image))), seedTree: input.seedDir === undefined ? "none" : hashTree(input.seedDir), baselineSkillTrees: input.delivery === "install" ? input.baselineSkills.map(hashTree) : [], diff --git a/src/engine/compare.ts b/src/engine/compare.ts index bec1851..70caec7 100644 --- a/src/engine/compare.ts +++ b/src/engine/compare.ts @@ -27,6 +27,13 @@ export interface ArmSummary { name: "baseline" | "proposed"; passes: number; totalRuns: number; + /** + * Runs whose grader reported no artifact. They are neither passes nor + * failures: passRate is passes over the remaining (graded) runs. Absent in + * cache records written before this field existed, which means 0. + */ + noArtifact?: number; + /** passes / graded runs (totalRuns minus noArtifact); 0 when no run was graded. */ passRate: number; totalCostUsd: number; runs: ArmRunSummary[]; @@ -108,7 +115,7 @@ export async function runCompare(options: CompareRunOptions): Promise= 0 ? "+" : ""}${formatPct(deltaPass)} pass | ${deltaCost >= 0 ? "+" : ""}$${deltaCost.toFixed(4)}`, ); @@ -157,6 +164,14 @@ export function formatCompareSummary(summary: CompareSummary): string { lines.push(" PASS: assertions satisfied"); } + // Pass rates silently computed over fewer runs than the header implies + // would hide an arm that stopped producing the artifact. + if ((caseSummary.baseline.noArtifact ?? 0) + (caseSummary.proposed.noArtifact ?? 0) > 0) { + lines.push( + ` NOTE: pass rates exclude runs with no artifact (baseline ${caseSummary.baseline.noArtifact ?? 0}/${caseSummary.baseline.totalRuns}, proposed ${caseSummary.proposed.noArtifact ?? 0}/${caseSummary.proposed.totalRuns})`, + ); + } + // A pass-rate delta that noise explains must not read like a receipt: // 1/3 → 2/3 passes the target assertion but is close to a coin flip. if ( @@ -174,7 +189,7 @@ export function formatCompareSummary(summary: CompareSummary): string { for (const arm of [caseSummary.baseline, caseSummary.proposed]) { for (const run of arm.runs) { if (run.pass) continue; - lines.push(` ${arm.name} run ${run.run} failed: ${run.grade.message}`); + lines.push(` ${arm.name} ${describeUnpassedRun(run)}`); lines.push(...graderEvidence(run.grade)); } } @@ -204,6 +219,28 @@ export function formatCompareSummary(summary: CompareSummary): string { return lines.join("\n"); } +/** + * "3/4 pass (75%), 1 no-artifact": the denominator is graded runs, so a run + * that produced nothing inflates neither the pass count nor the failure count. + */ +function formatPassCount(arm: ArmSummary): string { + const graded = gradedRuns(arm); + const base = `${arm.passes}/${graded} pass (${graded === 0 ? "n/a" : formatPct(arm.passRate)})`; + return arm.noArtifact ? `${base}, ${arm.noArtifact} no-artifact` : base; +} + +function describeUnpassedRun(run: ArmRunSummary): string { + if (!run.grade.noArtifact) return `run ${run.run} failed: ${run.grade.message}`; + // File graders already lead their message with "no artifact:". + return run.grade.message.startsWith("no artifact") + ? `run ${run.run} ${run.grade.message}` + : `run ${run.run} no artifact: ${run.grade.message}`; +} + +function gradedRuns(arm: ArmSummary): number { + return arm.totalRuns - (arm.noArtifact ?? 0); +} + function sha256(text: string): string { return createHash("sha256").update(text).digest("hex"); } @@ -283,7 +320,7 @@ export function validateRunnerSupport( if (effectiveTools(config, evalCase) !== "") { throw new Error( `runner "${runner.name}" is text-only, but scenario "${evalCase.name}" needs sandbox tools ` + - `(artifact mode, a command grader, or an explicit tools list) — use claude-p or make the scenario text-graded`, + `(artifact mode, a command or file grader, or an explicit tools list) — use claude-p or make the scenario text-graded`, ); } } @@ -363,11 +400,11 @@ export function formatMeasureSummary(summary: MeasureSummary): string { lines.push( "", caseSummary.name, - ` ${result.passes}/${result.totalRuns} pass (${formatPct(result.passRate)}) | $${result.totalCostUsd.toFixed(4)}`, + ` ${formatPassCount(result)} | $${result.totalCostUsd.toFixed(4)}`, ); for (const run of result.runs) { if (run.pass) continue; - lines.push(` run ${run.run} failed: ${run.grade.message}`); + lines.push(` ${describeUnpassedRun(run)}`); lines.push(...graderEvidence(run.grade)); } } @@ -526,12 +563,15 @@ async function runArm( } const passes = summaries.filter((summary) => summary.pass).length; + const noArtifact = summaries.filter((summary) => summary.grade.noArtifact).length; + const graded = runs - noArtifact; const totalCostUsd = summaries.reduce((sum, summary) => sum + summary.costUsd, 0); return { name: arm, passes, totalRuns: runs, - passRate: passes / runs, + noArtifact, + passRate: graded === 0 ? 0 : passes / graded, totalCostUsd, runs: summaries, }; @@ -661,6 +701,13 @@ function evaluateAssertions( return []; } + // With no graded run an arm's pass rate is not a measurement, and a + // directional verdict built on it would be invented. + const ungraded = [baseline, proposed].filter((arm) => gradedRuns(arm) === 0); + if (ungraded.length > 0) { + return ungraded.map((arm) => `${arm.name} produced no artifact in any run; there is nothing to compare`); + } + if (evalCase.kind === "target") { const failures = []; if (baseline.passRate >= 1) { @@ -679,9 +726,10 @@ function evaluateAssertions( } function inferMode(evalCase: EvalCaseConfig): "text" | "artifact" { - // text, json, and judge graders inspect the run's output only, so they + // Command and file graders read files the agent wrote in the sandbox; text, + // json, and judge graders inspect the run's output only, so they // demand no sandbox tools and stay valid on text-only runners (openai). - return evalCase.grader.type === "command" ? "artifact" : "text"; + return evalCase.grader.type === "command" || evalCase.grader.type === "file" ? "artifact" : "text"; } function defaultTools(mode: "text" | "artifact"): string { diff --git a/src/engine/config.ts b/src/engine/config.ts index f5793ac..f31c0d2 100644 --- a/src/engine/config.ts +++ b/src/engine/config.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import type { GraderSpec } from "./grader"; +import { assertGraderFileHasName } from "./grader-file"; import { parseAssertion } from "./json-assert"; import { resolveRenderVars, type RenderVars } from "./render"; import { deliveryValue, type Delivery } from "./skill-install"; @@ -400,6 +401,22 @@ function graderValue(value: unknown, scenarioName: string, baseDir: string): Gra expectExitCode: optionalNumber(value.expectExitCode, `${scenarioName}.grader.expectExitCode`), }; } + // The grader-file shape needs no "type": { "file": "./grade.eval.ts", "name": "..." }. + if (value.type === "file" || (value.type === undefined && value.file !== undefined)) { + const label = `${scenarioName}.grader`; + const file = resolveFrom(baseDir, requiredString(value.file, `${label}.file`)); + const name = requiredString(value.name, `${label}.name`); + // A missing file or misspelled name must fail at load time, not after the + // other arm's paid runs. + assertGraderFileHasName(file, name, label); + return { + type: "file", + file, + name, + cwd: stringValue(value.cwd, undefined), + timeoutMs: optionalNumber(value.timeoutMs, `${label}.timeoutMs`), + }; + } if (value.type === "judge") { const rubric = resolveFrom(baseDir, requiredString(value.rubric, `${scenarioName}.grader.rubric`)); // A missing rubric must fail at load time, not after the other arm's paid runs. @@ -421,7 +438,9 @@ function graderValue(value: unknown, scenarioName: string, baseDir: string): Gra minAccuracy, }; } - throw new Error(`${scenarioName}.grader.type must be "text", "json", "command", or "judge"`); + throw new Error( + `${scenarioName}.grader.type must be "text", "json", "command", or "judge" (or omit "type" and set "file" + "name" for a grader file)`, + ); } function kindValue(value: unknown, fallback: ScenarioKind): ScenarioKind { diff --git a/src/engine/grader-file.ts b/src/engine/grader-file.ts new file mode 100644 index 0000000..dc641a1 --- /dev/null +++ b/src/engine/grader-file.ts @@ -0,0 +1,151 @@ +import { existsSync, statSync } from "node:fs"; +import { resolve } from "node:path"; +import { plugin } from "bun"; +import * as packageEntry from "../index"; +import { evaluateGrader, formatGraderOutcome, graderOutcomePassed, isGraderFile, type GraderFile } from "../grade"; + +/** + * Reserved grader exit code meaning "the artifact to grade does not exist": + * no signal, neither a pass nor an assertion failure. Any command grader may + * exit with it; file graders exit with it when their artifact is missing. + * 77 is the automake "skipped" code, chosen because nothing else in the + * command-grader contract uses it. + */ +export const NO_ARTIFACT_EXIT_CODE = 77; + +/** Exit code for a grader file that cannot run at all (bad export, unknown name). */ +const GRADER_USAGE_EXIT_CODE = 2; + +/** The executable shim at the package root; file graders run as `bun grade ...`. */ +const PROMPTDIFF_BIN = resolve(import.meta.dir, "..", "..", "promptdiff"); + +export interface FileGraderSpec { + type: "file"; + /** Absolute path of the grader file. */ + file: string; + /** Which named grader in the file to run. */ + name: string; + cwd?: string; + timeoutMs?: number; +} + +/** + * The file grader compiled down to the command-grader contract: a shell + * command that runs in the sandbox, exits 0/1/77, and writes diagnostics to + * stderr. The same command can be run by hand inside a kept sandbox. + */ +export function fileGraderCommand(spec: FileGraderSpec): string { + return [process.execPath, PROMPTDIFF_BIN, "grade", "--file", spec.file, "--name", spec.name].map(shellQuote).join(" "); +} + +let aliasRegistered = false; + +/** + * Grader files import `grade` from "@theaiteam/promptdiff" (or "promptdiff"). + * Resolving that to this running copy means a grader file works without the + * package installed next to it, and always matches the runner's version. + */ +export function registerPackageAlias(): void { + if (aliasRegistered) return; + aliasRegistered = true; + plugin({ + name: "promptdiff-grader-alias", + setup(build) { + for (const specifier of ["@theaiteam/promptdiff", "promptdiff"]) { + build.module(specifier, () => ({ exports: { ...packageEntry }, loader: "object" })); + } + }, + }); +} + +export class GraderFileError extends Error {} + +/** Imports a grader file and returns its default export; throws GraderFileError on a bad shape. */ +export async function loadGraderFile(file: string): Promise { + registerPackageAlias(); + const path = resolve(file); + if (!existsSync(path)) { + throw new GraderFileError(`grader file not found: ${path}`); + } + const mod = (await import(path)) as { default?: unknown }; + if (!isGraderFile(mod.default)) { + throw new GraderFileError(`${path}: default export must be grade(artifactPath, { name: ({ result }) => ... })`); + } + return mod.default; +} + +export interface GraderFileRun { + exitCode: number; + stdout: string[]; + stderr: string[]; +} + +/** + * Runs one named grader. The artifact path resolves against `cwd`, which is + * the grader's working directory: the run sandbox, or the grader's `cwd` + * inside it, the same place a command grader would look. + */ +export async function runGraderFile(file: string, name: string, cwd: string = process.cwd()): Promise { + const loaded = await loadGraderFile(file); + const fn = loaded.graders[name]; + if (fn === undefined) { + return { + exitCode: GRADER_USAGE_EXIT_CODE, + stdout: [], + stderr: [`${resolve(file)}: no grader named ${JSON.stringify(name)} (has: ${Object.keys(loaded.graders).join(", ")})`], + }; + } + if (typeof loaded.artifact !== "string" || loaded.artifact.length === 0) { + return { + exitCode: GRADER_USAGE_EXIT_CODE, + stdout: [], + stderr: [`${resolve(file)}: grade() needs a non-empty artifact path, got ${JSON.stringify(loaded.artifact)}`], + }; + } + + const artifactPath = resolve(cwd, loaded.artifact); + if (!existsSync(artifactPath)) { + return { + exitCode: NO_ARTIFACT_EXIT_CODE, + stdout: [], + stderr: [`no artifact: ${loaded.artifact} does not exist in the grader's working directory`], + }; + } + if (statSync(artifactPath).isDirectory()) { + return { exitCode: 1, stdout: [], stderr: [`artifact ${loaded.artifact} is a directory, not a file`] }; + } + + const text = await Bun.file(artifactPath).text(); + const outcome = await evaluateGrader(fn, { path: loaded.artifact, text }); + const report = formatGraderOutcome(name, loaded.artifact, outcome); + return graderOutcomePassed(outcome) + ? { exitCode: 0, stdout: report, stderr: [] } + : { exitCode: 1, stdout: [], stderr: report }; +} + +/** + * Load-time check that `name` exists in `file`, run in a child process so a + * scenario load never executes grader code in the engine's own process. + * Throws with the scenario label on any problem, before any paid run. + */ +export function assertGraderFileHasName(file: string, name: string, label: string): void { + if (!existsSync(file)) { + throw new Error(`${label}.file: file not found: ${file}`); + } + const proc = Bun.spawnSync([process.execPath, PROMPTDIFF_BIN, "grade", "--file", file, "--list"], { + stdout: "pipe", + stderr: "pipe", + }); + if (proc.exitCode !== 0) { + const reason = proc.stderr.toString().trim() || `exit ${proc.exitCode}`; + throw new Error(`${label}.file: cannot load ${file}: ${reason}`); + } + const names = JSON.parse(proc.stdout.toString()) as string[]; + if (!names.includes(name)) { + throw new Error(`${label}.name: ${file} has no grader named ${JSON.stringify(name)} (has: ${names.join(", ")})`); + } +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} diff --git a/src/engine/grader.ts b/src/engine/grader.ts index 9d405bf..9f990f4 100644 --- a/src/engine/grader.ts +++ b/src/engine/grader.ts @@ -1,6 +1,7 @@ import { existsSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import type { RunResult } from "../types"; +import { fileGraderCommand, NO_ARTIFACT_EXIT_CODE, type FileGraderSpec } from "./grader-file"; import { evaluateAssertion, extractLastJson, parseAssertion } from "./json-assert"; import { gradeWithJudge, type JudgeGraderSpec } from "./judge"; @@ -23,6 +24,7 @@ export type GraderSpec = timeoutMs?: number; expectExitCode?: number; } + | FileGraderSpec | JudgeGraderSpec; export interface GradeInput { @@ -35,6 +37,12 @@ export interface GradeInput { export interface GradeResult { pass: boolean; + /** + * The grader reported that the artifact it grades does not exist (exit + * NO_ARTIFACT_EXIT_CODE). No signal: never a pass, and excluded from pass + * rates rather than counted as a failure. + */ + noArtifact?: boolean; message: string; stdout?: string; stderr?: string; @@ -63,9 +71,23 @@ export async function gradeRun(spec: GraderSpec, input: GradeInput): Promise { + const result = await gradeCommand( + { type: "command", command: fileGraderCommand(spec), cwd: spec.cwd, timeoutMs: spec.timeoutMs }, + sandboxDir, + outputFile, + { verdictFromStderr: true }, + ); + return result.pass ? { ...result, message: "file grader passed" } : result; +} + function gradeText(spec: Extract, output: string): GradeResult { for (const expected of spec.contains ?? []) { if (!output.includes(expected)) { @@ -107,6 +129,7 @@ async function gradeCommand( spec: Extract, sandboxDir: string, outputFile: string, + options: { verdictFromStderr?: boolean } = {}, ): Promise { const cwd = resolve(sandboxDir, spec.cwd ?? "."); if (!existsSync(cwd)) { @@ -115,7 +138,11 @@ async function gradeCommand( const proc = Bun.spawn(["sh", "-lc", spec.command], { cwd, - env: { ...process.env, PROMPTDIFF_OUTPUT_FILE: outputFile }, + env: { + ...process.env, + PROMPTDIFF_OUTPUT_FILE: outputFile, + PROMPTDIFF_NO_ARTIFACT_EXIT_CODE: String(NO_ARTIFACT_EXIT_CODE), + }, stdout: "pipe", stderr: "pipe", }); @@ -140,9 +167,23 @@ async function gradeCommand( } const expected = spec.expectExitCode ?? 0; + // File graders end stderr with a one-line verdict ("2 of 5 checks failed", + // "no artifact: ..."), which says more than the bare exit code. + const verdict = options.verdictFromStderr ? stderr.trim().split("\n").at(-1) || undefined : undefined; + // An explicit expectExitCode of 77 keeps its old meaning; only unexpected + // 77s are reclassified, so no existing grader flips between pass and fail. + if (code === NO_ARTIFACT_EXIT_CODE && expected !== NO_ARTIFACT_EXIT_CODE) { + return { + pass: false, + noArtifact: true, + message: verdict ?? `no artifact (grader exited ${NO_ARTIFACT_EXIT_CODE})`, + stdout, + stderr, + }; + } return { pass: code === expected, - message: code === expected ? "command grader passed" : `command exited ${code}, expected ${expected}`, + message: code === expected ? "command grader passed" : (verdict ?? `command exited ${code}, expected ${expected}`), stdout, stderr, }; diff --git a/src/engine/receipt.ts b/src/engine/receipt.ts index c84d082..e5d4efb 100644 --- a/src/engine/receipt.ts +++ b/src/engine/receipt.ts @@ -31,7 +31,8 @@ export interface Receipt { }; /** Hash of the rendered system prompt(s) — also pins render-var fixtures. */ renderedPromptSha256?: unknown; - results: Record; + /** noArtifact appears only when some run produced no artifact; passRate excludes those runs. */ + results: Record; samplingP?: number; productionModel?: string; } @@ -102,8 +103,14 @@ export function writeReceipts(dir: string, receipts: Receipt[]): string[] { }); } -function armResult(arm: { passes: number; totalRuns: number; passRate: number; totalCostUsd: number }) { - return { passes: arm.passes, totalRuns: arm.totalRuns, passRate: arm.passRate, costUsd: arm.totalCostUsd }; +function armResult(arm: { passes: number; totalRuns: number; noArtifact?: number; passRate: number; totalCostUsd: number }) { + return { + passes: arm.passes, + totalRuns: arm.totalRuns, + ...(arm.noArtifact ? { noArtifact: arm.noArtifact } : {}), + passRate: arm.passRate, + costUsd: arm.totalCostUsd, + }; } /** Repo-relative path + content hash; install-delivery skill dirs get a deterministic tree hash. */ diff --git a/src/engine/report.ts b/src/engine/report.ts index d48edba..ea22a17 100644 --- a/src/engine/report.ts +++ b/src/engine/report.ts @@ -43,6 +43,8 @@ function armRecord(arm: ArmSummary, config: ArmConfig): Record runner: config.runner, passes: arm.passes, totalRuns: arm.totalRuns, + // Present only when some run produced no artifact, so older records keep their shape. + ...(arm.noArtifact ? { noArtifact: arm.noArtifact } : {}), passRate: arm.passRate, costUsd: arm.totalCostUsd, }; diff --git a/src/grade.ts b/src/grade.ts new file mode 100644 index 0000000..dcf687b --- /dev/null +++ b/src/grade.ts @@ -0,0 +1,215 @@ +/** + * The grader file format: one TS/JS file whose default export is + * `grade(artifactPath, { name: ({ result }) => { ... } })`, referenced from a + * scenario as `"grader": { "file": "./grade.eval.ts", "name": "..." }`. + * + * This module is the authoring side: what a grader file imports. The runner + * side (reading the artifact, exit codes) lives in src/engine/grader-file.ts + * and compiles down to the command-grader contract. + */ + +/** What a named grader receives: the artifact under test. */ +export interface GradeContext { + result: ArtifactResult; +} + +export type GraderFn = (context: GradeContext) => void | Promise; + +/** + * The artifact plus the checks a grader runs against it. Every check records + * a pass or a failure and returns whether it passed; a failed check never + * stops the grader, so one run reports every failure at once. + */ +export interface ArtifactResult { + /** The artifact path as given to grade(), relative to the grader's cwd (the run sandbox). */ + readonly path: string; + /** The artifact's full contents. */ + readonly text: string; + /** + * The artifact parsed as JSON (parsed once, then cached). Invalid JSON + * fails the grader with the parse error; checks recorded before it stay. + */ + json(): T; + /** + * The general check: passes when `condition` is truthy. `message` says what + * went wrong; `context` (any JSON-serializable value) is printed with it, + * so a failure shows the data it was judged on. + */ + assert(condition: unknown, message: string, context?: unknown): boolean; + /** Passes when the artifact text contains `expected`. */ + shouldHave(expected: string): boolean; + /** Passes when the artifact text does not contain `forbidden`. */ + shouldNotHave(forbidden: string): boolean; + /** Passes when the artifact text matches `pattern`. */ + shouldMatch(pattern: RegExp): boolean; +} + +// Symbol.for, not Symbol(): a grader file and the runner may load separate +// copies of this module, and the brand must survive that. +const GRADER_FILE = Symbol.for("promptdiff.graderFile"); + +export interface GraderFile { + readonly [GRADER_FILE]: true; + /** Artifact path, relative to the grader's cwd (the run sandbox) unless absolute. */ + readonly artifact: string; + readonly graders: Readonly>; +} + +/** + * Declares the named graders for one artifact. The return value must be the + * file's default export. + */ +export function grade(artifact: string, graders: Record): GraderFile { + if (typeof graders !== "object" || graders === null || Array.isArray(graders)) { + throw new Error("grade() needs an object of named grader functions as its second argument"); + } + const names = Object.keys(graders); + if (names.length === 0) { + throw new Error("grade() needs at least one named grader"); + } + for (const name of names) { + if (typeof graders[name] !== "function") { + throw new Error(`grade(): grader ${JSON.stringify(name)} must be a function`); + } + } + // The artifact path is checked when a grader runs, not here: listing names + // at config load must work even when the path comes from an env var such as + // $PROMPTDIFF_OUTPUT_FILE, which only exists during a graded run. + return { [GRADER_FILE]: true, artifact, graders: { ...graders } }; +} + +export function isGraderFile(value: unknown): value is GraderFile { + return typeof value === "object" && value !== null && (value as Record)[GRADER_FILE] === true; +} + +export interface CheckFailure { + message: string; + /** JSON-rendered context or a description of what the artifact held instead. */ + detail?: string; +} + +export interface GraderOutcome { + checks: number; + failures: CheckFailure[]; +} + +const DETAIL_MAX_CHARS = 200; +const EXCERPT_CHARS = 60; + +/** + * Runs one named grader against artifact text and collects every check. + * A throw inside the grader (a bug, or json() on invalid JSON) is recorded as + * one more failure; the checks it made before throwing still count. + */ +export async function evaluateGrader(fn: GraderFn, artifact: { path: string; text: string }): Promise { + const outcome: GraderOutcome = { checks: 0, failures: [] }; + const record = (passed: boolean, failure: () => CheckFailure): boolean => { + outcome.checks += 1; + if (!passed) outcome.failures.push(failure()); + return passed; + }; + + const { text } = artifact; + let parsed: { value: unknown } | undefined; + const result: ArtifactResult = { + path: artifact.path, + text, + json() { + if (parsed === undefined) { + try { + parsed = { value: JSON.parse(text) }; + } catch (error) { + throw new ArtifactJsonError(error instanceof Error ? error.message : String(error)); + } + } + return parsed.value as any; + }, + assert(condition, message, context) { + return record(Boolean(condition), () => ({ + message, + detail: context === undefined ? undefined : `context: ${truncate(renderJson(context))}`, + })); + }, + shouldHave(expected) { + return record(text.includes(expected), () => ({ + message: `shouldHave(${JSON.stringify(expected)}): not found`, + detail: `artifact: ${text.length} chars, starts ${JSON.stringify(text.slice(0, EXCERPT_CHARS))}`, + })); + }, + shouldNotHave(forbidden) { + const at = text.indexOf(forbidden); + return record(at === -1, () => ({ + message: `shouldNotHave(${JSON.stringify(forbidden)}): found at offset ${at}`, + detail: `around it: ${JSON.stringify(excerptAround(text, at, forbidden.length))}`, + })); + }, + shouldMatch(pattern) { + // A global or sticky regex carries lastIndex between test() calls; a + // fresh copy keeps the check independent of earlier use. + const matched = new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, "")).test(text); + return record(matched, () => ({ + message: `shouldMatch(${pattern}): no match`, + detail: `artifact: ${text.length} chars, starts ${JSON.stringify(text.slice(0, EXCERPT_CHARS))}`, + })); + }, + }; + + try { + await fn({ result }); + } catch (error) { + // Counted as a check so "N of M checks failed" stays consistent. + outcome.checks += 1; + outcome.failures.push( + error instanceof ArtifactJsonError + ? { message: `artifact is not valid JSON: ${error.message}`, detail: `starts ${JSON.stringify(text.slice(0, EXCERPT_CHARS))}` } + : { message: `grader threw: ${error instanceof Error ? error.message : String(error)}` }, + ); + } + return outcome; +} + +/** + * Report lines for one graded artifact. The summary line comes last on + * purpose: compare/measure summaries show the tail of grader output, and the + * engine uses the last stderr line as the run's grade message. + */ +export function formatGraderOutcome(name: string, artifactPath: string, outcome: GraderOutcome): string[] { + const label = `grade ${JSON.stringify(name)} (${artifactPath})`; + if (outcome.checks === 0) { + return [`${label}: made no checks; a grader that checks nothing cannot pass`]; + } + if (outcome.failures.length === 0) { + return [`${label}: ${outcome.checks} of ${outcome.checks} checks passed`]; + } + const lines: string[] = []; + // One line per failure: summaries keep only the last few lines of grader + // output, and a two-line format would push the first failures out of view. + for (const failure of outcome.failures) { + lines.push(failure.detail === undefined ? `FAIL ${failure.message}` : `FAIL ${failure.message} | ${failure.detail}`); + } + lines.push(`${label}: ${outcome.failures.length} of ${outcome.checks} checks failed`); + return lines; +} + +export function graderOutcomePassed(outcome: GraderOutcome): boolean { + return outcome.checks > 0 && outcome.failures.length === 0; +} + +class ArtifactJsonError extends Error {} + +function renderJson(value: unknown): string { + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} + +function truncate(text: string): string { + return text.length <= DETAIL_MAX_CHARS ? text : `${text.slice(0, DETAIL_MAX_CHARS)}…`; +} + +function excerptAround(text: string, at: number, length: number): string { + const start = Math.max(0, at - 30); + return text.slice(start, at + length + 30); +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..b47d21f --- /dev/null +++ b/src/index.ts @@ -0,0 +1,10 @@ +/** + * Package entry point for grader files: + * + * import { grade } from "@theaiteam/promptdiff"; + * + * Grader files run under `promptdiff grade`, which also resolves the bare + * "promptdiff" specifier to this module. + */ +export { grade } from "./grade"; +export type { ArtifactResult, GradeContext, GraderFile, GraderFn } from "./grade"; diff --git a/tsconfig.json b/tsconfig.json index ae1d713..7383666 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,5 +8,5 @@ "types": ["bun-types"], "skipLibCheck": true }, - "include": ["src/**/*.ts", "test/**/*.ts", "promptdiff"] + "include": ["src/**/*.ts", "test/**/*.ts", "examples/**/*.eval.ts", "promptdiff"] } From e506245890f070d4e15fc4d169212114b813aecd Mon Sep 17 00:00:00 2001 From: Josh Owens Date: Wed, 30 Sep 2026 18:36:50 +0000 Subject: [PATCH 2/5] fix: point compare at measure when a scenario has no proposed arm compare on a baseline-only scenario failed with "compare requires proposed skill paths". The error now says to use `promptdiff measure`, which runs one instruction set without a proposed arm. Refs #32 Co-Authored-By: Claude Opus 5.5 --- src/engine/config.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/engine/config.ts b/src/engine/config.ts index f31c0d2..bfe28ad 100644 --- a/src/engine/config.ts +++ b/src/engine/config.ts @@ -327,7 +327,9 @@ function normalizeSkills(value: unknown, label: string, sharedSkills: string[] | if (sharedSkills !== undefined && (value === undefined || (isRecord(value) && value.skills === undefined))) { return sharedSkills; } - throw new Error(`compare requires ${label} skill paths`); + // A baseline-only scenario is a measurement, not a comparison; say so. + const hint = label === "proposed" ? "; to run one instruction set without a proposed arm, use `promptdiff measure`" : ""; + throw new Error(`compare requires ${label} skill paths${hint}`); } /** Resolves one arm's model/runner/baseUrl: per-arm override, per-arm field, shared override, shared field. */ From 34e64de45a4c04f876a8a70e89a2a49112994907 Mon Sep 17 00:00:00 2001 From: Josh Owens Date: Wed, 30 Sep 2026 18:36:50 +0000 Subject: [PATCH 3/5] test: grader files, no-artifact accounting, and the measure hint Covers check collection (every failure reported, thrown errors and invalid JSON recorded), exit-code mapping, artifact paths resolved against the grader cwd, command graders exiting 77, scenario validation of the grader-file shape (missing file, unknown name, bad export), measure and compare summary lines with no-artifact runs, the grader file in the cache key, report records, and the compare error that points at measure. Refs #32 Co-Authored-By: Claude Opus 5.5 --- test/grade-file.test.ts | 388 ++++++++++++++++++++++++++++++++++++++++ test/report.test.ts | 9 + 2 files changed, 397 insertions(+) create mode 100644 test/grade-file.test.ts diff --git a/test/grade-file.test.ts b/test/grade-file.test.ts new file mode 100644 index 0000000..a8923cb --- /dev/null +++ b/test/grade-file.test.ts @@ -0,0 +1,388 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, expect, test } from "bun:test"; +import { evaluateGrader, formatGraderOutcome, grade, graderOutcomePassed } from "../src/grade"; +import { buildCacheKey } from "../src/engine/cache"; +import { formatCompareSummary, formatMeasureSummary, runCompare, runMeasure } from "../src/engine/compare"; +import { loadCompareConfig } from "../src/engine/config"; +import { gradeRun } from "../src/engine/grader"; +import { NO_ARTIFACT_EXIT_CODE, runGraderFile } from "../src/engine/grader-file"; +import type { Runner, RunnerRunOptions } from "../src/types"; + +const root = mkdtempSync(join(tmpdir(), "promptdiff-grade-file-")); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +const run = { output: "", costUsd: 0, turns: 1, durationMs: 1, models: [], raw: {} }; + +// A grader file in the shape the docs lead with: a semantic assert first, +// sugar after. It imports the package by name, which the runner aliases. +const GRADER_SOURCE = ` +import { grade } from "@theaiteam/promptdiff"; + +export default grade("out/plan.json", { + "covers-8": ({ result }) => { + const kept = result.json().a_roll; + result.assert( + kept.some((s) => s.source_in <= 8 && 8 <= s.source_out), + "no kept range covers t=8.0", + { ranges: kept }, + ); + result.shouldHave("cinema cutie"); + result.shouldNotHave("lorem"); + }, + "bare-name": ({ result }) => { + result.shouldMatch(/cutie/); + }, +}); +`; + +function writeGraderFile(dir: string, source = GRADER_SOURCE): string { + mkdirSync(dir, { recursive: true }); + const file = join(dir, "grade.eval.ts"); + writeFileSync(file, source, "utf8"); + return file; +} + +function writeArtifact(sandbox: string, plan: unknown): void { + mkdirSync(join(sandbox, "out"), { recursive: true }); + writeFileSync(join(sandbox, "out", "plan.json"), JSON.stringify(plan), "utf8"); +} + +const PASSING_PLAN = { a_roll: [{ source_in: 0, source_out: 9 }], note: "cinema cutie" }; +const FAILING_PLAN = { a_roll: [{ source_in: 0, source_out: 7.5 }], note: "lorem ipsum" }; + +test("evaluateGrader collects every failed check instead of stopping at the first", async () => { + const outcome = await evaluateGrader( + ({ result }) => { + result.assert(false, "first semantic failure", { seen: [1, 2] }); + result.shouldHave("absent"); + result.shouldNotHave("lorem"); + result.shouldMatch(/nope/); + result.assert(true, "passes"); + }, + { path: "a.txt", text: "lorem ipsum" }, + ); + + expect(outcome.checks).toBe(5); + expect(outcome.failures.map((failure) => failure.message)).toEqual([ + "first semantic failure", + 'shouldHave("absent"): not found', + 'shouldNotHave("lorem"): found at offset 0', + "shouldMatch(/nope/): no match", + ]); + expect(outcome.failures[0]?.detail).toBe('context: {"seen":[1,2]}'); + expect(outcome.failures[1]?.detail).toContain('starts "lorem ipsum"'); + expect(graderOutcomePassed(outcome)).toBe(false); +}); + +test("each check returns whether it passed, so graders can branch on it", async () => { + const seen: boolean[] = []; + await evaluateGrader( + ({ result }) => { + seen.push(result.shouldHave("ok"), result.shouldNotHave("ok"), result.assert(1, "truthy"), result.shouldMatch(/o+k/g)); + // A global regex must not carry lastIndex into a second check. + seen.push(result.shouldMatch(/o+k/g)); + }, + { path: "a.txt", text: "ok" }, + ); + expect(seen).toEqual([true, false, true, true, true]); +}); + +test("invalid JSON and thrown errors become failures; earlier checks still count", async () => { + const badJson = await evaluateGrader( + ({ result }) => { + result.shouldHave("x"); + result.json(); + result.shouldHave("never reached"); + }, + { path: "a.json", text: "x not json" }, + ); + expect(badJson.checks).toBe(2); + expect(badJson.failures).toHaveLength(1); + expect(badJson.failures[0]?.message).toStartWith("artifact is not valid JSON:"); + + const thrown = await evaluateGrader( + async ({ result }) => { + result.assert(false, "recorded before the throw"); + throw new Error("kept is undefined"); + }, + { path: "a.json", text: "{}" }, + ); + expect(thrown.failures.map((failure) => failure.message)).toEqual([ + "recorded before the throw", + "grader threw: kept is undefined", + ]); +}); + +test("a grader that checks nothing does not pass", async () => { + const outcome = await evaluateGrader(() => {}, { path: "a.txt", text: "anything" }); + expect(graderOutcomePassed(outcome)).toBe(false); + expect(formatGraderOutcome("empty", "a.txt", outcome)).toEqual([ + 'grade "empty" (a.txt): made no checks; a grader that checks nothing cannot pass', + ]); +}); + +test("the report puts one line per failure and the verdict last", async () => { + const outcome = await evaluateGrader( + ({ result }) => { + result.assert(false, "no kept range covers t=8.0", { ranges: [[0, 7.5]] }); + result.shouldHave("cinema"); + result.assert(true, "fine"); + }, + { path: "plan.json", text: "{}" }, + ); + const lines = formatGraderOutcome("covers-8", "plan.json", outcome); + expect(lines).toEqual([ + 'FAIL no kept range covers t=8.0 | context: {"ranges":[[0,7.5]]}', + 'FAIL shouldHave("cinema"): not found | artifact: 2 chars, starts "{}"', + 'grade "covers-8" (plan.json): 2 of 3 checks failed', + ]); +}); + +test("grade() rejects an empty or non-function grader map", () => { + expect(() => grade("a.json", {})).toThrow(/at least one named grader/); + expect(() => grade("a.json", { x: "nope" as never })).toThrow(/"x" must be a function/); +}); + +test("runGraderFile resolves the artifact against the grader cwd and maps outcomes to exit codes", async () => { + const dir = join(root, "run-grader-file"); + const file = writeGraderFile(dir); + const sandbox = join(dir, "sandbox"); + mkdirSync(sandbox, { recursive: true }); + + const missing = await runGraderFile(file, "covers-8", sandbox); + expect(missing.exitCode).toBe(NO_ARTIFACT_EXIT_CODE); + expect(missing.stderr.join("\n")).toContain("no artifact: out/plan.json does not exist"); + + writeArtifact(sandbox, FAILING_PLAN); + const failing = await runGraderFile(file, "covers-8", sandbox); + expect(failing.exitCode).toBe(1); + expect(failing.stderr).toHaveLength(4); + expect(failing.stderr.at(-1)).toBe('grade "covers-8" (out/plan.json): 3 of 3 checks failed'); + + writeArtifact(sandbox, PASSING_PLAN); + const passing = await runGraderFile(file, "covers-8", sandbox); + expect(passing.exitCode).toBe(0); + expect(passing.stdout).toEqual(['grade "covers-8" (out/plan.json): 3 of 3 checks passed']); + + const unknown = await runGraderFile(file, "nope", sandbox); + expect(unknown.exitCode).toBe(2); + expect(unknown.stderr[0]).toContain('no grader named "nope" (has: covers-8, bare-name)'); +}); + +test("a file grader runs as a command in the sandbox and reports no-artifact distinctly", async () => { + const dir = join(root, "grade-run"); + const file = writeGraderFile(dir); + const sandbox = join(dir, "sandbox"); + mkdirSync(sandbox, { recursive: true }); + const spec = { type: "file" as const, file, name: "covers-8", timeoutMs: 30_000 }; + + const missing = await gradeRun(spec, { run, sandboxDir: sandbox }); + expect(missing).toMatchObject({ pass: false, noArtifact: true }); + expect(missing.message).toStartWith("no artifact: out/plan.json does not exist"); + + writeArtifact(sandbox, FAILING_PLAN); + const failing = await gradeRun(spec, { run, sandboxDir: sandbox }); + expect(failing.pass).toBe(false); + expect(failing.noArtifact).toBeUndefined(); + // The verdict line replaces "command exited 1"; the failures stay in stderr. + expect(failing.message).toBe('grade "covers-8" (out/plan.json): 3 of 3 checks failed'); + expect(failing.stderr).toContain("FAIL no kept range covers t=8.0"); + + writeArtifact(sandbox, PASSING_PLAN); + const passing = await gradeRun(spec, { run, sandboxDir: sandbox }); + expect(passing).toMatchObject({ pass: true, message: "file grader passed" }); +}); + +test("command graders signal no-artifact with the reserved exit code", async () => { + const sandbox = join(root, "command-no-artifact"); + mkdirSync(sandbox, { recursive: true }); + const script = 'test -f draft.json || exit "$PROMPTDIFF_NO_ARTIFACT_EXIT_CODE"; grep -q ok draft.json'; + + const missing = await gradeRun({ type: "command", command: script }, { run, sandboxDir: sandbox }); + expect(missing).toMatchObject({ pass: false, noArtifact: true, message: "no artifact (grader exited 77)" }); + + writeFileSync(join(sandbox, "draft.json"), "not it", "utf8"); + const failing = await gradeRun({ type: "command", command: script }, { run, sandboxDir: sandbox }); + expect(failing).toMatchObject({ pass: false, message: "command exited 1, expected 0" }); + expect(failing.noArtifact).toBeUndefined(); + + // An explicit expectExitCode of 77 keeps meaning "77 is a pass". + const expected = await gradeRun({ type: "command", command: "exit 77", expectExitCode: 77 }, { run, sandboxDir: sandbox }); + expect(expected).toMatchObject({ pass: true }); + expect(expected.noArtifact).toBeUndefined(); +}); + +function writeScenario(dir: string, grader: unknown, extra: Record = {}): string { + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "agent.md"), "Agent.", "utf8"); + writeFileSync(join(dir, "skill.md"), "Skill.", "utf8"); + const path = join(dir, "scenario.json"); + writeFileSync( + path, + JSON.stringify({ + agent: "./agent.md", + skills: ["./skill.md"], + model: "sonnet", + runs: 4, + scenarios: [{ name: "probe", prompt: "write the plan", grader }], + ...extra, + }), + "utf8", + ); + return path; +} + +test("scenario loading accepts the grader-file shape and resolves the file from the scenario", () => { + const dir = join(root, "config-ok"); + writeGraderFile(dir); + const path = writeScenario(dir, { file: "./grade.eval.ts", name: "covers-8", timeoutMs: 5_000 }); + const config = loadCompareConfig(path, {}, { singleArm: true }); + expect(config.cases[0]?.grader).toEqual({ + type: "file", + file: join(dir, "grade.eval.ts"), + name: "covers-8", + cwd: undefined, + timeoutMs: 5_000, + }); + // An explicit "type": "file" is the same shape. + const typed = writeScenario(join(dir, "typed"), { type: "file", file: "../grade.eval.ts", name: "bare-name" }); + expect(loadCompareConfig(typed, {}, { singleArm: true }).cases[0]?.grader).toMatchObject({ type: "file", name: "bare-name" }); +}); + +test("scenario loading rejects a missing grader file, an unknown name, and a bad default export", () => { + const dir = join(root, "config-bad"); + writeGraderFile(dir); + writeGraderFile(join(dir, "plain"), "export default { graders: {} };\n"); + + const load = (grader: unknown) => () => loadCompareConfig(writeScenario(dir, grader), {}, { singleArm: true }); + expect(load({ file: "./missing.eval.ts", name: "x" })).toThrow(/probe\.grader\.file: file not found: .*missing\.eval\.ts/); + expect(load({ file: "./grade.eval.ts", name: "covers-9" })).toThrow( + /probe\.grader\.name: .* has no grader named "covers-9" \(has: covers-8, bare-name\)/, + ); + expect(load({ file: "./grade.eval.ts" })).toThrow(/probe\.grader\.name must be a non-empty string/); + expect(load({ file: "./plain/grade.eval.ts", name: "x" })).toThrow(/default export must be grade\(/); + expect(load({ type: "nope" })).toThrow(/or omit "type" and set "file" \+ "name"/); +}); + +test("compare on a baseline-only scenario points at measure", () => { + const dir = join(root, "compare-hint"); + writeGraderFile(dir); + // The reproduction-probe shape: one instruction set, no proposed arm. + const path = writeScenario( + dir, + { file: "./grade.eval.ts", name: "covers-8" }, + { skills: undefined, baselineSkills: ["./skill.md"] }, + ); + expect(() => loadCompareConfig(path)).toThrow( + "compare requires proposed skill paths; to run one instruction set without a proposed arm, use `promptdiff measure`", + ); +}); + +/** Writes the plan into the sandbox on the listed (1-based) calls only; the rest produce nothing. */ +function artifactRunner(plans: Array): Runner { + let call = 0; + return { + name: "mock", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, + async run(options: RunnerRunOptions) { + const plan = plans[call % plans.length]; + call += 1; + if (plan !== undefined) writeArtifact(options.cwd, plan); + return { output: "done", costUsd: 0.01, turns: 1, durationMs: 1, models: ["m"], raw: {} }; + }, + }; +} + +test("measure reports no-artifact runs apart from passes and failures", async () => { + const dir = join(root, "measure"); + writeGraderFile(dir); + const path = writeScenario(dir, { file: "./grade.eval.ts", name: "covers-8" }); + const config = loadCompareConfig(path, { sandboxRoot: join(dir, "runs") }, { singleArm: true }); + + // The field report from the issue: four real passes and one run that wrote + // nothing. The honest tally is 4/4 pass, 1 no-artifact, not 4/5. + const summary = await runMeasure({ + config: { ...config, runs: 5 }, + runner: artifactRunner([PASSING_PLAN, PASSING_PLAN, undefined, PASSING_PLAN, PASSING_PLAN]), + }); + const result = summary.cases[0]!.result; + expect(result).toMatchObject({ passes: 4, totalRuns: 5, noArtifact: 1, passRate: 1 }); + + const text = formatMeasureSummary(summary); + expect(text).toContain("4/4 pass (100%), 1 no-artifact"); + expect(text).toContain("run 3 no artifact: out/plan.json does not exist"); + expect(text).not.toContain("failed"); +}); + +test("measure with failures and no-artifact runs keeps the two apart", async () => { + const dir = join(root, "measure-mixed"); + writeGraderFile(dir); + const path = writeScenario(dir, { file: "./grade.eval.ts", name: "covers-8" }); + const config = loadCompareConfig(path, { sandboxRoot: join(dir, "runs"), runs: 3 }, { singleArm: true }); + + const summary = await runMeasure({ config, runner: artifactRunner([PASSING_PLAN, FAILING_PLAN, undefined]) }); + const text = formatMeasureSummary(summary); + expect(text).toContain("1/2 pass (50%), 1 no-artifact"); + expect(text).toContain('run 2 failed: grade "covers-8" (out/plan.json): 3 of 3 checks failed'); + // The per-failure lines reach the summary as grader evidence. + expect(text).toContain("FAIL no kept range covers t=8.0 | context:"); + expect(text).toContain("run 3 no artifact:"); +}); + +test("compare excludes no-artifact runs from pass rates, notes them, and refuses an arm with none graded", async () => { + const dir = join(root, "compare"); + writeGraderFile(dir); + writeFileSync(join(dir, "proposed.md"), "PROPOSED", "utf8"); + const path = writeScenario(dir, { file: "./grade.eval.ts", name: "covers-8" }, { proposedSkills: ["./proposed.md"] }); + const config = loadCompareConfig(path, { sandboxRoot: join(dir, "runs"), runs: 2 }); + const regression = { ...config, cases: config.cases.map((evalCase) => ({ ...evalCase, kind: "regression" as const })) }; + + // Baseline passes twice; proposed passes once and writes nothing once. + const plans = [PASSING_PLAN, PASSING_PLAN, PASSING_PLAN, undefined]; + const shared = artifactRunner(plans); + const summary = await runCompare({ config: regression, runners: { baseline: shared, proposed: shared } }); + const caseSummary = summary.cases[0]!; + expect(caseSummary.proposed).toMatchObject({ passes: 1, noArtifact: 1, passRate: 1 }); + // 1/1 is not a regression from 2/2; a no-artifact run is not a failure. + expect(caseSummary.assertions).toEqual([]); + const text = formatCompareSummary(summary); + expect(text).toContain("proposed: 1/1 pass (100%), 1 no-artifact"); + expect(text).toContain("NOTE: pass rates exclude runs with no artifact (baseline 0/2, proposed 1/2)"); + expect(text).toContain("proposed run 2 no artifact:"); + + // An arm that never produced the artifact has no pass rate to compare. + const empty = artifactRunner([undefined]); + const none = await runCompare({ config: regression, runners: { baseline: shared, proposed: empty } }); + expect(none.cases[0]!.assertions).toEqual(["proposed produced no artifact in any run; there is nothing to compare"]); + expect(formatCompareSummary(none)).toContain("proposed: 0/0 pass (n/a), 2 no-artifact"); +}); + +test("the baseline cache key covers the grader file's content", () => { + const dir = join(root, "cache-key"); + const file = writeGraderFile(dir); + const keyInput = { + systemPrompt: "s", + casePrompt: "p", + arm: { model: "sonnet", runner: "claude-p" as const }, + runs: 2, + tools: "default", + mode: "artifact" as const, + delivery: "inline" as const, + grader: { type: "file" as const, file, name: "covers-8" }, + images: [], + baselineSkills: [], + }; + const before = buildCacheKey(keyInput); + writeGraderFile(dir, GRADER_SOURCE.replace("t=8.0", "t=9.0")); + expect(buildCacheKey(keyInput)).not.toBe(before); +}); + +test("grade --help documents the exit codes", () => { + const result = Bun.spawnSync(["./promptdiff", "grade", "--help"], { stdout: "pipe", stderr: "pipe" }); + const stdout = result.stdout.toString(); + expect(result.exitCode).toBe(0); + expect(stdout).toContain("usage: promptdiff grade --file (--name | --list)"); + expect(stdout).toContain("77 no artifact"); +}); diff --git a/test/report.test.ts b/test/report.test.ts index 81e741b..dec1aaa 100644 --- a/test/report.test.ts +++ b/test/report.test.ts @@ -62,3 +62,12 @@ test("appendNdjsonReport appends across invocations and creates parent dirs", () rmSync(dir, { recursive: true, force: true }); } }); + +test("report records carry a noArtifact count only when some run produced no artifact", () => { + const summary = makeSummary(); + const scope = summary.cases[0]!; + scope.proposed = { ...scope.proposed, totalRuns: 4, noArtifact: 1 }; + const [record] = buildReportRecords(summary, "2026-01-01T00:00:00.000Z"); + expect((record?.proposed as Record).noArtifact).toBe(1); + expect(record?.baseline).not.toHaveProperty("noArtifact"); +}); From 334f9cfdf7dc693fb748fbb53c13de63abb2d04a Mon Sep 17 00:00:00 2001 From: Josh Owens Date: Wed, 30 Sep 2026 18:36:50 +0000 Subject: [PATCH 4/5] docs: document grader files and the no-artifact outcome README and SPEC describe the grader file format, leading with result.assert for semantic checks, how a file grader runs as a command grader, the exit-77 no-artifact convention for command graders, and how measure and compare count no-artifact runs. examples/07-grader-file adds two semantic graders with sample artifacts that run without a model call. Refs #32 Co-Authored-By: Claude Opus 5.5 --- README.md | 136 +++++++++++++++++- SPEC.md | 56 +++++++- examples/07-grader-file/README.md | 111 ++++++++++++++ examples/07-grader-file/agent.md | 7 + examples/07-grader-file/grade.eval.ts | 41 ++++++ .../samples/lunch-clash/schedule.json | 6 + .../07-grader-file/samples/pass/schedule.json | 6 + examples/07-grader-file/scenario.json | 21 +++ examples/07-grader-file/skill.md | 11 ++ examples/README.md | 12 +- 10 files changed, 396 insertions(+), 11 deletions(-) create mode 100644 examples/07-grader-file/README.md create mode 100644 examples/07-grader-file/agent.md create mode 100644 examples/07-grader-file/grade.eval.ts create mode 100644 examples/07-grader-file/samples/lunch-clash/schedule.json create mode 100644 examples/07-grader-file/samples/pass/schedule.json create mode 100644 examples/07-grader-file/scenario.json create mode 100644 examples/07-grader-file/skill.md diff --git a/README.md b/README.md index bd4eaa9..a74bb60 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,18 @@ `promptdiff` is a small Bun CLI for testing whether an LLM prompt or skill change actually changes behavior — with any model, not just Claude. -It has four commands: +It has five commands: - `run`: one bounded model invocation with an agent file and optional inlined skill files. - `compare`: N-run baseline-vs-proposed comparison from a JSON scenario file, - with deterministic text or command graders (plus calibrated LLM judges). + with deterministic text, json, command, or grader-file graders (plus + calibrated LLM judges). - `measure`: N-run single-arm characterization — pass rates, no assertions. - `calibrate`: prove a judge grader against labeled rubric fixtures before compare/measure will let it grade anything. +- `grade`: run one named grader from a grader file by hand, for example + inside a kept sandbox. Model access goes through pluggable runners. Two ship today: @@ -327,6 +330,11 @@ Reading results: - Declare `"productionModel": "gpt-5.5"` and any arm testing a different model gets a warning on the summary — a pass on the wrong model validates prompt logic, not production behavior. +- Runs whose grader reported [no artifact](#no-artifact) are left out of the + pass rate and counted on their own (`1/1 pass (100%), 1 no-artifact`), + with a `NOTE` naming each arm's count. An arm with no graded run at all + fails a target or regression assertion, since it has no pass rate to + compare. ### Raw per-run results @@ -455,13 +463,23 @@ per-case pass rates, no delta and no assertions: ``` The scenario file is the compare format — render vars, images, pricing, -`productionModel`, and both grader types all apply — but only `skills` (or +`productionModel`, and every grader type apply — but only `skills` (or `baselineSkills`) is required; no proposed arm. Don't fake this with identical compare arms: two identical arms at small n routinely produce verdicts like `FAIL: proposed regressed below baseline` out of pure sampling noise. `measure` exits 0 whenever the runs complete — a measurement has no pass/fail. +A run where the agent wrote nothing is not a pass and not a reproduction. +When the grader reports [no artifact](#no-artifact), the summary line counts +it separately, and the pass rate covers only the graded runs: + +``` +reproduces-defect + 4/4 pass (100%), 1 no-artifact | $0.4210 + run 3 no artifact: 02_debating/edit_plan.draft.json does not exist in the grader's working directory +``` + ## Comparing models The A/B variable does not have to be the skill text. Hold the prompt and @@ -675,6 +693,113 @@ scenario so no tools are demanded, and script against the file: } ``` +### Grader files + +A grader file holds named graders for one artifact. Most real graders are +semantic, so lead with `result.assert`: plain code for the condition, a +message for when it fails, and the data it judged as context. + +```ts +// grade.eval.ts +import { grade } from "@theaiteam/promptdiff"; + +type Plan = { a_roll: Array<{ source_in: number; source_out: number }> }; +const keptSeconds = (plan: Plan) => plan.a_roll.reduce((sum, s) => sum + (s.source_out - s.source_in), 0); + +export default grade("02_debating/edit_plan.draft.json", { + "product-named": ({ result }) => { + const kept: Plan["a_roll"] = result.json().a_roll; + result.assert( + kept.some((s) => s.source_in <= 8.0 && 8.0 <= s.source_out), + "no kept range covers t=8.0", + { ranges: kept }, + ); + result.shouldHave("cinema cutie"); + result.shouldNotHave("lorem"); + }, + "still-edits": ({ result }) => { + const kept = keptSeconds(result.json()); + result.assert(kept / 64.672 <= 0.9, `kept ${kept}s of 64.672s: keep-everything degeneration`); + }, +}); +``` + +The scenario names the file and the grader: + +```json +"grader": { "file": "./grade.eval.ts", "name": "product-named" } +``` + +`result` offers: + +- `result.assert(condition, message, context?)`: passes when `condition` is + truthy. On failure it prints `message` and `context` as JSON. Use it for + anything a substring cannot express: ranges, counts, ratios, structure. +- `result.json()`: the artifact parsed as JSON. Invalid JSON fails the + grader with the parse error. +- `result.text` and `result.path`: the raw contents and the path as given. +- `result.shouldHave(str)`, `result.shouldNotHave(str)`, + `result.shouldMatch(regex)`: literal checks. A failure prints what the + artifact held instead (its start, or the text around a forbidden match). + +Every check returns whether it passed and never stops the grader, so one +run reports every failure. An exception in the grader (a bug, or `json()` on +invalid JSON) is one more failure; checks made before it still count. A +grader that makes no checks fails. + +How it runs: + +- The artifact path resolves against the grader's working directory: the + run sandbox, or the grader's `cwd` inside it, the same place a command + grader runs. Absolute paths work too. To grade the run's final text output + instead of a file, pass `process.env.PROMPTDIFF_OUTPUT_FILE!`. +- A file grader is a command grader underneath. The engine runs + `promptdiff grade --file --name ` in the sandbox, with the + same exit-code contract: 0 pass, 1 failed checks (all printed to stderr), + 77 [no artifact](#no-artifact). Run that command yourself inside a + `--keep-sandbox` directory to debug a grader without new model runs. +- The grader's last stderr line (`grade "product-named" (...): 2 of 3 checks + failed`) becomes the run's message in the summary, followed by the + failure lines. +- The file and the name are checked when the scenario loads, before any paid + run: a missing file, a default export that is not `grade(...)`, or an + unknown name fails with the names the file does define. +- Optional fields: `"cwd"` and `"timeoutMs"`, as for command graders. Like + command graders, grader files need a tool-capable runner (claude-p) unless + the scenario sets `"mode": "text"`. +- The import resolves to the running promptdiff under `promptdiff grade`, so + the file works without the package installed beside it. `import { grade } + from "promptdiff"` works too. Install `@theaiteam/promptdiff` as a dev + dependency for editor types. +- `compare --cache` keys on the grader file's content, so editing a check + misses the cache. Files the grader imports are not hashed. + +[examples/07-grader-file](./examples/07-grader-file/) runs two semantic +graders against sample schedules with no model run. + +### No artifact + +A run where the agent wrote nothing gives a grader nothing to judge. That +is its own outcome, `no-artifact`, distinct from a failed check: counting it +as a failure inflates reproduction rates, and a negative check +(`shouldNotHave`) against a missing file would otherwise pass. + +- File graders report it on their own when the artifact path does not exist. +- Command graders report it by exiting 77. The code is also in + `$PROMPTDIFF_NO_ARTIFACT_EXIT_CODE`: + + ```sh + test -f draft.json || exit "$PROMPTDIFF_NO_ARTIFACT_EXIT_CODE" + ``` + + A command grader with `"expectExitCode": 77` keeps treating 77 as a pass. + Before this existed, exit 77 was an ordinary failure; it is still never a + pass. + +`measure` and `compare` exclude no-artifact runs from pass rates and show +the count on the summary line (`4/4 pass (100%), 1 no-artifact`). Reports +and receipts carry a `noArtifact` count when it is non-zero. + ## Judge graders (calibrated) Some judgments cannot be expressed as string checks — "is this the @@ -762,8 +887,9 @@ Lessons from production compare runs, for scenario authors: ## Security note Command graders execute arbitrary shell commands from scenario files (inside -the per-run sandbox, but with your local permissions). Only run scenario files -you trust. +the per-run sandbox, but with your local permissions). Grader files are code +too: loading a scenario imports each one to check its grader names. Only run +scenario files you trust. ## Development diff --git a/SPEC.md b/SPEC.md index b74d0e5..12f2501 100644 --- a/SPEC.md +++ b/SPEC.md @@ -244,6 +244,54 @@ output is also written into the sandbox and exposed to command graders as `$PROMPTDIFF_OUTPUT_FILE`, so completion-only runs can be command-graded (scenario `mode: "text"` keeps the tools demand at zero). +**Grader files.** The command contract (run in the sandbox, exit code is the +verdict, stderr is the diagnostics) stays the base layer; any executable in +any language can grade. A grader file is a layer on top of it for the common +case of checking one artifact from TS/JS. The file's default export is +`grade(artifactPath, { name: ({ result }) => { ... } })`, and a scenario +selects one grader with `{ "file": "./grade.eval.ts", "name": "..." }` (no +`type` needed; `"type": "file"` is accepted). `result` offers +`assert(condition, message, context?)` for semantic checks, `json()`, `text`, +`path`, and the literal checks `shouldHave`, `shouldNotHave`, and +`shouldMatch`. `assert` is the primary check in docs and examples on +purpose: the strongest graders in use are semantic (range coverage, +calibrated regex rulers), and substring sugar alone would push authors +toward brittle string matching. + +The engine compiles a file grader to a command grader: +` grade --file --name `, run in the same +sandbox cwd with the same timeout handling. `promptdiff grade` resolves the +artifact against its cwd (the sandbox, or the grader's `cwd` inside it), +reads it, runs the named grader, and collects every check rather than +stopping at the first failure. It exits 0 when all checks pass, 1 when any +fail (each failure on one stderr line with its context, then a one-line +verdict that becomes the run's grade message), 77 when the artifact does not +exist, and 2 when the file or name is unusable. A grader that makes no +checks fails. An exception inside a grader is recorded as one more failure. +Under `promptdiff grade`, the specifiers `@theaiteam/promptdiff` and +`promptdiff` resolve to the running copy through a Bun runtime plugin, so a +grader file needs no local install and cannot drift from the runner's +version. Scenario loading checks each file and name in a child process +(`promptdiff grade --list`) before any paid run. The baseline cache key +includes the grader file's content hash (not the files it imports). + +**No artifact.** A grader that finds nothing to grade reports `no-artifact`, +a third outcome beside pass and assertion-failed. File graders report it +when the artifact path does not exist. Command graders report it by exiting +77 (also exported as `$PROMPTDIFF_NO_ARTIFACT_EXIT_CODE`), unless their +`expectExitCode` is 77, which keeps its old meaning. No existing command +grader changes between pass and fail: an exit 77 was a failure before and is +still not a pass. A no-artifact run is never a pass, so a negative check +cannot pass on an agent that produced nothing, and it is not counted as a +failure, so reproduction rates are not inflated by runs with no signal. +`ArmSummary.noArtifact` counts these runs; `passRate` is passes over graded +runs (`totalRuns - noArtifact`), and Fisher's p uses the graded counts. +Summaries print `4/4 pass (100%), 1 no-artifact` and list each such run. +`compare` adds a `NOTE` with both arms' no-artifact counts whenever either +is non-zero, and fails target and regression assertions for an arm with no +graded run at all, which has no pass rate to compare. An arm that produces +the artifact less often is noted, not failed. + LLM judges are implemented, WITH mandatory calibration, for judgments that cannot be expressed as local checks (semantic and style rubrics that regex both under- and over-catches). The design premise is that an uncalibrated @@ -282,6 +330,8 @@ promptdiff # Bun executable shim src/cli.ts # command parsing and user-facing orchestration src/args.ts # strict local flag parser src/prompt.ts # frontmatter stripping and prompt assembly +src/index.ts # package entry for grader files (exports grade) +src/grade.ts # grader-file authoring API and check collection src/runner/ # provider-coupled runner code src/engine/ # compare loop, config loading, graders, sandbox lifecycle test/ # Bun tests @@ -310,8 +360,10 @@ always runs fresh. Deleting the cache directory busts it. decisive effects at small N, but there is no statistical test yet. - Scenario authoring is manual JSON. A future tuning loop should generate these files from accepted findings. -- Command graders run trusted local commands from scenario files. Do not run - untrusted scenario files. +- Command graders run trusted local commands from scenario files, and loading + a scenario imports its grader files. Do not run untrusted scenario files. +- The cache key hashes a grader file but not modules it imports; editing a + helper module alone does not bust the baseline cache. - The openai runner is a single completion per run: no tool use, so it can only answer text-graded questions. Tool-using evals on non-Claude models would need an agentic runner (e.g. wrapping another agent CLI). diff --git a/examples/07-grader-file/README.md b/examples/07-grader-file/README.md new file mode 100644 index 0000000..e09582a --- /dev/null +++ b/examples/07-grader-file/README.md @@ -0,0 +1,111 @@ +# Grader file + +A grader file replaces a hand-written grader script: one TS file, named +graders, and a `result` object with checks. promptdiff reads the artifact, +parses the JSON, runs every check, prints each failure with the data it was +judged on, and turns the outcome into the command-grader exit code. + +The skill under test plans four meetings into one day and writes +`schedule.json`. The two graders in [`grade.eval.ts`](./grade.eval.ts) are +semantic: "no two meetings overlap" and "nothing lands in the lunch block" +are time-range math, which no substring check can express. So both lead +with `result.assert(condition, message, context)`, and `shouldHave` appears +once, for the one literal thing worth checking (Priya's name). + +```ts +export default grade("schedule.json", { + "lunch-stays-free": ({ result }) => { + const meetings: Meeting[] = result.json().meetings; + const clashes = meetings.filter((m) => minutes(m.start) < minutes("13:00") && minutes(m.end) > minutes("12:00")); + result.assert(clashes.length === 0, "a meeting overlaps the 12:00-13:00 lunch block", { clashes }); + }, +}); +``` + +The scenario references each grader by name: + +```json +"grader": { "file": "./grade.eval.ts", "name": "lunch-stays-free" } +``` + +**Cost:** free for the grader runs below. The `measure` run (8 haiku runs +in artifact mode) was not run for this README, so its cost is not measured. +**Requires:** bun; the claude CLI for `measure` + +## Try the graders without a model run + +`promptdiff grade` is the command every file grader compiles to. It resolves +the artifact path against the current directory, which inside a real run is +the sandbox. The `samples/` directories hold hand-written schedules, so you +can check a grader before paying for any runs. + +A clean schedule: + +```bash +cd examples/07-grader-file/samples/pass +../../../../promptdiff grade --file ../../grade.eval.ts --name no-overlaps +``` + +``` +grade "no-overlaps" (schedule.json): 6 of 6 checks passed +``` + +A schedule with two defects. Each grader reports its own, with the meetings +that caused it: + +```bash +cd examples/07-grader-file/samples/lunch-clash +../../../../promptdiff grade --file ../../grade.eval.ts --name no-overlaps +``` + +``` +FAIL 1:1 with Priya starts before Design review ends | context: {"before":{"title":"Design review","start":"09:00","end":"10:00"},"after":{"title":"1:1 with Priya","start":"09:30","end":"10:00"}} +grade "no-overlaps" (schedule.json): 1 of 6 checks failed +``` + +```bash +../../../../promptdiff grade --file ../../grade.eval.ts --name lunch-stays-free +``` + +``` +FAIL a meeting overlaps the 12:00-13:00 lunch block | context: {"clashes":[{"title":"Hiring debrief","start":"12:00","end":"13:00"}]} +grade "lunch-stays-free" (schedule.json): 1 of 1 checks failed +``` + +No schedule at all (run from `samples/`, which has no `schedule.json`): + +```bash +cd examples/07-grader-file/samples +../../../promptdiff grade --file ../grade.eval.ts --name no-overlaps; echo $? +``` + +``` +no artifact: schedule.json does not exist in the grader's working directory +77 +``` + +## Run it for real + +```bash +./promptdiff measure --scenario ./examples/07-grader-file/scenario.json +``` + +This output was not captured for this README. Each scenario prints a line +like `3/3 pass (100%), 1 no-artifact`. + +## What to notice + +- **Every failure is reported, not the first one.** A failed check records + itself and the grader keeps going, so one run shows every defect. +- **No artifact is not a failure.** Exit 77 means the agent wrote nothing. + `measure` and `compare` leave those runs out of the pass rate and count + them on their own (`3/3 pass (100%), 1 no-artifact`), so a run that + produced nothing can neither pass a negative check nor count as a + reproduced defect. +- **Context is the evidence.** The third argument to `result.assert` is + printed on failure. Pass the data the check looked at, and the summary + answers "why did run 2 fail" without a `--keep-sandbox` re-run. +- The import is `@theaiteam/promptdiff`. Under `promptdiff grade` it + resolves to the running promptdiff, so the grader works without + installing the package next to it; install it as a dev dependency if you + want editor types. diff --git a/examples/07-grader-file/agent.md b/examples/07-grader-file/agent.md new file mode 100644 index 0000000..e8a1a91 --- /dev/null +++ b/examples/07-grader-file/agent.md @@ -0,0 +1,7 @@ +--- +name: scheduler +description: Example agent for promptdiff's grader-file example. +--- + +You are a scheduling assistant. You work in the current directory and write +your results to files there. Follow the skill instructions exactly. diff --git a/examples/07-grader-file/grade.eval.ts b/examples/07-grader-file/grade.eval.ts new file mode 100644 index 0000000..d1fa836 --- /dev/null +++ b/examples/07-grader-file/grade.eval.ts @@ -0,0 +1,41 @@ +import { grade } from "@theaiteam/promptdiff"; + +interface Meeting { + title: string; + start: string; + end: string; +} + +const minutes = (time: string) => { + const [hours, mins] = time.split(":").map(Number); + return hours * 60 + mins; +}; + +export default grade("schedule.json", { + "no-overlaps": ({ result }) => { + const meetings: Meeting[] = result.json().meetings; + const sorted = [...meetings].sort((a, b) => minutes(a.start) - minutes(b.start)); + // Semantic checks first: this is range math that no substring can express. + for (let i = 1; i < sorted.length; i += 1) { + const [before, after] = [sorted[i - 1], sorted[i]]; + result.assert( + minutes(after.start) >= minutes(before.end), + `${after.title} starts before ${before.title} ends`, + { before, after }, + ); + } + result.assert( + sorted.every((m) => minutes(m.start) >= minutes("09:00") && minutes(m.end) <= minutes("17:00")), + "a meeting falls outside 09:00-17:00", + { meetings: sorted }, + ); + result.assert(meetings.length === 4, `expected 4 meetings, got ${meetings.length}`); + // Sugar for the literal cases. + result.shouldHave("Priya"); + }, + "lunch-stays-free": ({ result }) => { + const meetings: Meeting[] = result.json().meetings; + const clashes = meetings.filter((m) => minutes(m.start) < minutes("13:00") && minutes(m.end) > minutes("12:00")); + result.assert(clashes.length === 0, "a meeting overlaps the 12:00-13:00 lunch block", { clashes }); + }, +}); diff --git a/examples/07-grader-file/samples/lunch-clash/schedule.json b/examples/07-grader-file/samples/lunch-clash/schedule.json new file mode 100644 index 0000000..999db66 --- /dev/null +++ b/examples/07-grader-file/samples/lunch-clash/schedule.json @@ -0,0 +1,6 @@ +{"meetings": [ + {"title": "Design review", "start": "09:00", "end": "10:00"}, + {"title": "1:1 with Priya", "start": "09:30", "end": "10:00"}, + {"title": "Budget sync", "start": "11:00", "end": "11:45"}, + {"title": "Hiring debrief", "start": "12:00", "end": "13:00"} +]} diff --git a/examples/07-grader-file/samples/pass/schedule.json b/examples/07-grader-file/samples/pass/schedule.json new file mode 100644 index 0000000..44926db --- /dev/null +++ b/examples/07-grader-file/samples/pass/schedule.json @@ -0,0 +1,6 @@ +{"meetings": [ + {"title": "Design review", "start": "09:00", "end": "10:00"}, + {"title": "1:1 with Priya", "start": "10:15", "end": "10:45"}, + {"title": "Budget sync", "start": "11:00", "end": "11:45"}, + {"title": "Hiring debrief", "start": "13:00", "end": "14:00"} +]} diff --git a/examples/07-grader-file/scenario.json b/examples/07-grader-file/scenario.json new file mode 100644 index 0000000..bb2f97a --- /dev/null +++ b/examples/07-grader-file/scenario.json @@ -0,0 +1,21 @@ +{ + "name": "day-planner", + "agent": "./agent.md", + "skills": ["./skill.md"], + "model": "haiku", + "runs": 4, + "mode": "artifact", + "maxTurns": 6, + "scenarios": [ + { + "name": "no-overlaps", + "prompt": "Plan Tuesday between 09:00 and 17:00: Design review (60 min), 1:1 with Priya (30 min), Budget sync (45 min), Hiring debrief (60 min). Keep 12:00-13:00 free for lunch.", + "grader": { "file": "./grade.eval.ts", "name": "no-overlaps" } + }, + { + "name": "lunch-stays-free", + "prompt": "Plan Tuesday between 09:00 and 17:00: Design review (60 min), 1:1 with Priya (30 min), Budget sync (45 min), Hiring debrief (60 min). Keep 12:00-13:00 free for lunch.", + "grader": { "file": "./grade.eval.ts", "name": "lunch-stays-free" } + } + ] +} diff --git a/examples/07-grader-file/skill.md b/examples/07-grader-file/skill.md new file mode 100644 index 0000000..42801cf --- /dev/null +++ b/examples/07-grader-file/skill.md @@ -0,0 +1,11 @@ +--- +name: day-planner +--- + +Plan the requested meetings into one working day and write the plan to +`schedule.json` in the current directory, shaped like: + +{"meetings": [{"title": "...", "start": "HH:MM", "end": "HH:MM"}]} + +Use 24-hour times. Give each meeting exactly its requested length. Meetings +must not overlap. diff --git a/examples/README.md b/examples/README.md index 0585bad..cf06397 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,13 +1,15 @@ # Examples -Six runnable, self-contained examples, ordered as a learning curve. Each +Seven runnable, self-contained examples, ordered as a learning curve. Each directory holds everything it needs (agent, skills, scenario file) plus its own README with the exact command, an honest cost label, and its output — so you can see what promptdiff gives you before spending anything yourself. The output in 01-04 and 06 is captured from a real run. 05 is the exception: it needs a local ollama server, which was not available where these examples -were built, so its README shows an illustrative block and says so. +were built, so its README shows an illustrative block and says so. 07's +grader output is real and free (it grades sample files with no model run); +its `measure` output was not captured. Run them from the repo root. All except 05 need the `claude` CLI installed; none need an API key beyond what `claude` already uses. @@ -20,6 +22,8 @@ none need an API key beyond what `claude` already uses. | [04-judge-grader](./04-judge-grader/) | A calibrated LLM judge end to end: rubric, labeled fixtures, `calibrate`, then a compare — and the refuse-to-grade gate when calibration is missing. | ~$0.10 | | [05-any-model](./05-any-model/) | The `openai` runner against local ollama: not Claude-only, zero API keys. Same shape works for vLLM, llama.cpp, OpenRouter. Output illustrative, not a real run. | free (local) | | [06-token-economics](./06-token-economics/) | Turn caps (`--max-turns`) + raw per-run results (`--raw-out`): token-category analysis, and how a capped run is failed without grading. | ~$0.19 | +| [07-grader-file](./07-grader-file/) | A grader file: named graders with `result.assert` for semantic checks, every failure reported with its context, and `no-artifact` (exit 77) kept apart from failures. Try the graders on sample files before any paid run. | free (graders); `measure` not measured | -Every `scenario.json` here is loaded by `test/examples.test.ts` in CI, so the -examples can't silently drift from the config schema. +Every `scenario.json` here is loaded by `test/examples.test.ts` in CI, and +every `*.eval.ts` grader file is typechecked, so the examples can't silently +drift from the config schema or the grader API. From 13f2a72e5d7516e5857ea6aec6813bc1435a027f Mon Sep 17 00:00:00 2001 From: Josh Owens Date: Wed, 30 Sep 2026 19:47:28 +0000 Subject: [PATCH 5/5] Address PR #45 review feedback (pass 1) - Fix: look up grader names by own property, so "constructor" or "toString" exit 2 instead of running an inherited function - Fix: leave samplingP undefined when an arm has no graded runs - Fix: kill the grader's whole process group on timeout; a hung file grader (sh -> bun) kept the pipes open past timeoutMs - Test: inherited names, samplingP with graded denominators, file grader timeout Addresses review comments from github-actions. Co-Authored-By: Claude Opus 5.5 --- src/engine/compare.ts | 8 +++++- src/engine/grader-file.ts | 3 +- src/engine/grader.ts | 15 ++++++++-- test/grade-file.test.ts | 60 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/engine/compare.ts b/src/engine/compare.ts index 70caec7..430752e 100644 --- a/src/engine/compare.ts +++ b/src/engine/compare.ts @@ -115,7 +115,7 @@ export async function runCompare(options: CompareRunOptions): Promise { const loaded = await loadGraderFile(file); - const fn = loaded.graders[name]; + // Own keys only: "constructor" or "toString" must not resolve to Object.prototype. + const fn = Object.hasOwn(loaded.graders, name) ? loaded.graders[name] : undefined; if (fn === undefined) { return { exitCode: GRADER_USAGE_EXIT_CODE, diff --git a/src/engine/grader.ts b/src/engine/grader.ts index 9f990f4..b14f266 100644 --- a/src/engine/grader.ts +++ b/src/engine/grader.ts @@ -145,14 +145,25 @@ async function gradeCommand( }, stdout: "pipe", stderr: "pipe", + // Own process group, so a timeout can kill the shell's children too. A + // file grader is `sh -> bun`; killing only the shell leaves bun holding + // the output pipes open until it exits on its own. + detached: true, }); + const kill = (signal: NodeJS.Signals) => { + try { + process.kill(-proc.pid, signal); + } catch { + proc.kill(signal); + } + }; let timedOut = false; const timeoutMs = spec.timeoutMs ?? 120_000; const timeout = setTimeout(() => { timedOut = true; - proc.kill("SIGTERM"); - setTimeout(() => proc.kill("SIGKILL"), 2_000); + kill("SIGTERM"); + setTimeout(() => kill("SIGKILL"), 2_000); }, timeoutMs); const [stdout, stderr, code] = await Promise.all([ diff --git a/test/grade-file.test.ts b/test/grade-file.test.ts index a8923cb..9a30f3e 100644 --- a/test/grade-file.test.ts +++ b/test/grade-file.test.ts @@ -7,6 +7,7 @@ import { buildCacheKey } from "../src/engine/cache"; import { formatCompareSummary, formatMeasureSummary, runCompare, runMeasure } from "../src/engine/compare"; import { loadCompareConfig } from "../src/engine/config"; import { gradeRun } from "../src/engine/grader"; +import { fisherExactTwoTailedP } from "../src/engine/stats"; import { NO_ARTIFACT_EXIT_CODE, runGraderFile } from "../src/engine/grader-file"; import type { Runner, RunnerRunOptions } from "../src/types"; @@ -171,6 +172,45 @@ test("runGraderFile resolves the artifact against the grader cwd and maps outcom expect(unknown.stderr[0]).toContain('no grader named "nope" (has: covers-8, bare-name)'); }); +test("inherited Object.prototype names are not graders", async () => { + const dir = join(root, "inherited-names"); + const file = writeGraderFile(dir); + const sandbox = join(dir, "sandbox"); + mkdirSync(sandbox, { recursive: true }); + writeArtifact(sandbox, PASSING_PLAN); + + for (const name of ["constructor", "toString", "hasOwnProperty"]) { + const result = await runGraderFile(file, name, sandbox); + expect(result.exitCode).toBe(2); + expect(result.stderr[0]).toContain(`no grader named "${name}"`); + } + // A scenario naming one fails at load, before any paid run. + const path = writeScenario(dir, { file: "./grade.eval.ts", name: "constructor" }); + expect(() => loadCompareConfig(path, {}, { singleArm: true })).toThrow(/has no grader named "constructor"/); +}); + +test("a file grader that hangs is killed at timeoutMs and fails as a timeout, not no-artifact", async () => { + const dir = join(root, "timeout"); + const file = writeGraderFile( + dir, + `import { grade } from "@theaiteam/promptdiff"; +export default grade("out/plan.json", { + hang: async () => { await new Promise((r) => setTimeout(r, 10_000)); }, +}); +`, + ); + const sandbox = join(dir, "sandbox"); + mkdirSync(sandbox, { recursive: true }); + writeArtifact(sandbox, PASSING_PLAN); + + const started = Date.now(); + const result = await gradeRun({ type: "file", file, name: "hang", timeoutMs: 1_500 }, { run, sandboxDir: sandbox }); + expect(result.pass).toBe(false); + expect(result.noArtifact).toBeUndefined(); + expect(result.message).toBe("grader timed out after 1500ms"); + expect(Date.now() - started).toBeLessThan(8_000); +}, 15_000); + test("a file grader runs as a command in the sandbox and reports no-artifact distinctly", async () => { const dir = join(root, "grade-run"); const file = writeGraderFile(dir); @@ -359,6 +399,26 @@ test("compare excludes no-artifact runs from pass rates, notes them, and refuses expect(formatCompareSummary(none)).toContain("proposed: 0/0 pass (n/a), 2 no-artifact"); }); +test("compare leaves samplingP undefined when an arm has no graded runs, and uses graded denominators otherwise", async () => { + const dir = join(root, "sampling-p"); + writeGraderFile(dir); + writeFileSync(join(dir, "proposed.md"), "PROPOSED", "utf8"); + const path = writeScenario(dir, { file: "./grade.eval.ts", name: "covers-8" }, { proposedSkills: ["./proposed.md"] }); + const config = loadCompareConfig(path, { sandboxRoot: join(dir, "runs"), runs: 2 }); + + const shared = artifactRunner([PASSING_PLAN, PASSING_PLAN]); + const none = await runCompare({ config, runners: { baseline: shared, proposed: artifactRunner([undefined]) } }); + expect(none.cases[0]!.samplingP).toBeUndefined(); + + // Baseline 2/2 graded, proposed 1/1 graded plus one no-artifact run. + const mixed = await runCompare({ + config, + runners: { baseline: artifactRunner([PASSING_PLAN]), proposed: artifactRunner([PASSING_PLAN, undefined]) }, + }); + expect(mixed.cases[0]!.proposed).toMatchObject({ passes: 1, noArtifact: 1 }); + expect(mixed.cases[0]!.samplingP).toBe(fisherExactTwoTailedP(2, 2, 1, 1)); +}); + test("the baseline cache key covers the grader file's content", () => { const dir = join(root, "cache-key"); const file = writeGraderFile(dir);