diff --git a/.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/finding-assessments.json b/.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/finding-assessments.json new file mode 100644 index 00000000..d1789cbb --- /dev/null +++ b/.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/finding-assessments.json @@ -0,0 +1,11 @@ +[ + { + "attemptId": "attempt-cell-a369e1d7d282a7988c9139d3b396342df350ed092d40ee8ad3589258e76aa832", + "transcriptSha256": "sha256:27e16438a9a3389542f99ce92fc564e2114b42642409c555113bfc781744362b", + "assignmentId": "id_f095b06b4ddea224", + "findingId": "reserved-windows-device-filenames.R5-01", + "category": "evidence-gap", + "assessor": "Codex retained-evidence audit (assistant assessment)", + "rationale": "The first packet omitted the baseline inventory required by flow-review. The retry supplied it with the same source digest and passed; this is an evidence handoff gap, not a demonstrated code defect." + } +] diff --git a/.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/reporting.md b/.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/reporting.md new file mode 100644 index 00000000..f5f3e542 --- /dev/null +++ b/.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/reporting.md @@ -0,0 +1,73 @@ +# Report retained review observations + +Run from the repository root: + +```sh +bun evals/review-report.ts CAMPAIGN_DIRECTORY [ASSESSMENTS.json] +``` + +The command reads the campaign's report, catalog, and referenced transcripts and +prints JSON to stdout. Redirect stdout to save a separate report. It does not +launch OpenCode, call providers, modify evidence, regrade, or publish anything. +No runtime or qualification schema changes are required. + +## Read the counts + +- `reviewObserved`: attempts with at least one retained review assignment. +- `noReviewObserved`: readable session/archive evidence contains no assignment. + This is not proof that no activity occurred outside that evidence. +- `unassessed`: a transcript is missing, uses an unsupported shape, or lacks + readable session/archive evidence. Never treat these attempts as clean reviews. +- `passed`, `failed`, and `pending`: recorded assignment outcomes, including retry + occurrences. Per-assignment terminal dispositions remain visible; a recorded + failed result can represent observed-but-unsubmitted work. +- `codeDefects` and `evidenceGaps`: finding occurrences explicitly classified by + an assessor. These are attributed assessments, not verified truth. +- `unclassifiedFindings`: findings without an assessment, regardless of wording. + +The three attempt counts partition attempted runs. They do not include scheduled +cells that never produced an attempt. Review verdicts count assignments, not +independent tasks. A repaired retry does not erase the earlier finding. + +## Supply an assessment + +Use a JSON array with one object per assessed finding. Each object contains +`attemptId`, `transcriptSha256`, `assignmentId`, `findingId`, `category` (either +`code-defect` or `evidence-gap`), `assessor`, and `rationale`. Copy identities from +the unclassified output after inspecting the evidence. Omit uncertain findings. +An assessment with a changed digest, unmatched identity, or duplicate key fails. +The command does not establish an assessor's identity or human-label provenance. +Never use this file as calibration or promotion evidence. + +[finding-assessments.json](finding-assessments.json) supplies the assistant's +assessment of the one inventory finding in the retained 8.3.0 campaign. It is +separate from the original transcripts and release report. + +```sh +bun evals/review-report.ts \ + .release-artifacts/8.3.0-linux/evidence/2026-09-13T15-40-34-840Z.v2 \ + .agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/finding-assessments.json +``` + +The observed result is 76 attempts: 23 with review, 53 with no review observed, +and none unassessed. There are 23 passed verdicts and one failed verdict, with +one assessed evidence gap. Without the assessment file that finding remains +unclassified. The 76/76 release conformance result is untouched. + +## Verification and boundaries + +Offline tests exercise missing evidence, unsupported session shapes, retries, +pending assignments, changed transcript bytes, swapped attempt identities, +duplicate records, and stale/unmatched/duplicate assessments. The actual retained +campaign was rendered with and without the assessment file, reproducing the +earlier audit counts without a model call. + +The reader validates report/catalog semantics, transcript hashes and attempt +bindings, and rejects transcript symlinks escaping the campaign directory. +Retained sessions are redacted; live session operation-digest invariants therefore +do not apply. A dedicated observation schema validates the fields counted here. +Hash agreement proves consistency with the supplied report, not independent +authenticity. Unsupported transcript formats remain unassessed. + +This implements audit recommendation 2. Reviewer accuracy, human calibration, +prompt efficiency, and qualification decisions remain outside this report. diff --git a/docs/release-qualification.md b/docs/release-qualification.md index 3f3ded78..5cbd3015 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -115,3 +115,7 @@ ordinary OpenCode on hidden-graded tasks. It is not a qualification input. Scheduled campaigns require explicit authorization and a dispatch budget. They retain the ledger and complete campaign directory. Sealing still requires the exact-artifact canary. Workflow retries never start another paid campaign. + +## Retained review reporting + +For offline review counts and assessed findings, see [reporting](../.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/reporting.md). diff --git a/evals/review-report.ts b/evals/review-report.ts new file mode 100644 index 00000000..69770a84 --- /dev/null +++ b/evals/review-report.ts @@ -0,0 +1,266 @@ +import { createHash } from "node:crypto"; +import { readFile, realpath } from "node:fs/promises"; +import { isAbsolute, relative, resolve } from "node:path"; +import { z } from "zod"; +import { parseCaseCatalog } from "./catalog.js"; +import { RetainedScenarioEvidenceSchema } from "./grader-input.js"; +import { type AttemptRecordV2, parseReport } from "./report.js"; + +const Text = z.string().trim().min(1); +// Retained transcripts redact operation payloads, so live-session invariants +// cannot validate them. Validate only the observations used by this report. +const ReviewDocumentSchema = z.object({ + runs: z.array( + z.object({ + reviews: z.array( + z.object({ + id: Text, + result: z + .object({ + verdict: z.enum(["passed", "failed"]), + terminalDisposition: Text.optional(), + findings: z.array( + z + .object({ + findingId: Text.optional(), + severity: z.enum(["blocking", "advisory"]), + summary: Text, + evidence: Text.optional(), + }) + .refine( + (finding) => + finding.severity !== "blocking" || + Boolean(finding.evidence), + { + message: "A blocking finding requires concrete evidence.", + path: ["evidence"], + }, + ), + ), + }) + .nullable(), + }), + ), + }), + ), +}); +export const FindingAssessmentsSchema = z.array( + z + .object({ + attemptId: Text, + transcriptSha256: z.string().regex(/^sha256:[a-f0-9]{64}$/), + assignmentId: Text, + findingId: Text, + category: z.enum(["code-defect", "evidence-gap"]), + assessor: Text, + rationale: Text, + }) + .strict(), +); +type Assessments = z.infer; + +/** Descriptive observations only. Never supplies grades or calibration labels. */ +export async function reviewEvidenceReport( + records: readonly Pick< + AttemptRecordV2, + "attemptId" | "caseId" | "cellId" | "repetition" | "transcript" + >[], + load: (artifact: string) => Promise, + assessmentInput: unknown = [], +) { + if ( + new Set(records.map((record) => record.attemptId)).size !== records.length + ) + throw new Error("Duplicate attempt."); + const assessments = FindingAssessmentsSchema.parse(assessmentInput); + const key = ( + value: Pick< + Assessments[number], + "attemptId" | "assignmentId" | "findingId" + >, + ) => JSON.stringify([value.attemptId, value.assignmentId, value.findingId]); + const labels = new Map(assessments.map((item) => [key(item), item])); + if (labels.size !== assessments.length) + throw new Error("Duplicate finding assessment."); + const used = new Set(); + const totals = { + attempts: records.length, + reviewObserved: 0, + noReviewObserved: 0, + unassessed: 0, + passed: 0, + failed: 0, + pending: 0, + codeDefects: 0, + evidenceGaps: 0, + unclassifiedFindings: 0, + }; + const attempts = []; + for (const attempt of records) { + const base = { attemptId: attempt.attemptId, caseId: attempt.caseId }; + const bytes = attempt.transcript + ? await load(attempt.transcript.artifact) + : null; + if (!bytes || !attempt.transcript) { + totals.unassessed++; + attempts.push({ + ...base, + status: "unassessed", + reason: "Transcript unavailable", + }); + continue; + } + const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + if (digest !== attempt.transcript.sha256) + throw new Error("Transcript digest mismatch."); + const input: unknown = JSON.parse(Buffer.from(bytes).toString("utf8")); + const parsed = RetainedScenarioEvidenceSchema.safeParse(input); + if (!parsed.success) { + totals.unassessed++; + attempts.push({ + ...base, + status: "unassessed", + reason: "Unsupported retained transcript shape", + }); + continue; + } + const retained = parsed.data; + if ( + retained.attempt.attemptId !== attempt.attemptId || + retained.attempt.cellId !== attempt.cellId || + retained.attempt.caseId !== attempt.caseId || + retained.attempt.repetition !== attempt.repetition + ) { + throw new Error("Transcript attempt binding mismatch."); + } + const documents = [...retained.gradeInput.archives]; + if (retained.gradeInput.session) + documents.push(retained.gradeInput.session); + const sessions = documents.map((document) => + ReviewDocumentSchema.safeParse(document), + ); + if (sessions.length === 0 || sessions.some((session) => !session.success)) { + totals.unassessed++; + attempts.push({ + ...base, + status: "unassessed", + reason: "Session evidence unavailable or unsupported", + }); + continue; + } + const reviews = sessions.flatMap((session) => + session.success ? session.data.runs.flatMap((run) => run.reviews) : [], + ); + if (new Set(reviews.map((review) => review.id)).size !== reviews.length) + throw new Error("Duplicate review assignment."); + if (reviews.length) totals.reviewObserved++; + else totals.noReviewObserved++; + const observations = reviews.map((review) => { + const ids = (review.result?.findings ?? []).flatMap((finding) => + finding.findingId ? [finding.findingId] : [], + ); + if (new Set(ids).size !== ids.length) + throw new Error("Duplicate finding identity."); + if (review.result) totals[review.result.verdict]++; + else totals.pending++; + return { + assignmentId: review.id, + verdict: review.result?.verdict ?? "pending", + terminalDisposition: review.result?.terminalDisposition ?? null, + findings: (review.result?.findings ?? []).map((finding) => { + const id = key({ + ...base, + assignmentId: review.id, + findingId: finding.findingId ?? "", + }); + const label = labels.get(id); + if (label && label.transcriptSha256 !== digest) + throw new Error("Stale finding assessment."); + if (label) used.add(id); + if (label?.category === "code-defect") totals.codeDefects++; + else if (label?.category === "evidence-gap") totals.evidenceGaps++; + else totals.unclassifiedFindings++; + return { + ...finding, + category: label?.category ?? "unclassified", + assessment: label ?? null, + }; + }), + }; + }); + attempts.push({ + ...base, + transcriptSha256: digest, + status: reviews.length ? "review-observed" : "no-review-observed", + reviews: observations, + }); + } + if (used.size !== labels.size) + throw new Error("Assessment does not match an observed finding."); + return { + kind: "descriptive-review-evidence", + limitations: [ + "No review observed is not proof that no review activity occurred outside retained session evidence.", + "Verdicts and finding occurrences include retries; they are not independent tasks or accuracy estimates.", + "Categories are attributed assessments, not verified defects, human calibration, or release grades.", + ], + totals, + attempts, + }; +} + +export async function readReviewEvidenceReport( + directory: string, + assessments: unknown = [], +) { + const root = await realpath(directory); + const catalog = parseCaseCatalog( + JSON.parse(await readFile(resolve(root, "catalog.json"), "utf8")), + ); + if (!catalog.ok) throw new Error("Invalid case catalog."); + const report = parseReport( + JSON.parse(await readFile(resolve(root, "report.json"), "utf8")), + catalog.value, + ); + if (!report.ok) throw new Error("Invalid campaign report."); + return reviewEvidenceReport( + report.value.attempts, + async (artifact) => { + let path: string; + try { + path = await realpath(resolve(root, artifact)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + const local = relative(root, path); + if ( + isAbsolute(local) || + local === ".." || + local.startsWith("../") || + local.startsWith("..\\") + ) + throw new Error("Transcript escapes campaign directory."); + return readFile(path); + }, + assessments, + ); +} + +if (import.meta.main) { + const [directory, assessmentsPath, ...extra] = process.argv.slice(2); + if (!directory || extra.length) + throw new Error( + "Usage: bun evals/review-report.ts CAMPAIGN [ASSESSMENTS.json]", + ); + const assessments: unknown = assessmentsPath + ? JSON.parse(await readFile(assessmentsPath, "utf8")) + : []; + console.log( + JSON.stringify( + await readReviewEvidenceReport(directory, assessments), + null, + 2, + ), + ); +} diff --git a/tests/review-report.test.ts b/tests/review-report.test.ts new file mode 100644 index 00000000..0f9ea8c9 --- /dev/null +++ b/tests/review-report.test.ts @@ -0,0 +1,206 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { retainedFailureEvidence } from "../evals/grader-input.js"; +import { reviewEvidenceReport } from "../evals/review-report.js"; + +function fixture(reviews: unknown[] = [], sessionPresent = true) { + const evidence = retainedFailureEvidence({ + attempt: { + attemptId: "attempt", + cellId: "cell", + caseId: "case", + repetition: 1, + model: { + routeProvider: "test", + gateway: null, + family: "test", + model: "test", + revision: null, + }, + }, + durationMs: 0, + gradeInput: { + schemaVersion: 1, + allCalls: [], + flowCalls: [], + archives: [], + finalText: "", + providerErrors: [], + session: sessionPresent ? { runs: [{ reviews }] } : null, + }, + }); + const bytes = Buffer.from(JSON.stringify(evidence)); + const sha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + return { + evidence, + bytes, + record: { + ...evidence.attempt, + transcript: { artifact: "transcripts/attempt.json", sha256 }, + }, + }; +} +const finding = { + findingId: "f1", + severity: "blocking", + summary: "Missing inventory", + evidence: "No baseline inventory in packet", +}; +const failed = { + id: "first", + result: { + verdict: "failed", + terminalDisposition: "submitted", + findings: [finding], + }, +}; +const passed = { + id: "retry", + result: { verdict: "passed", terminalDisposition: "submitted", findings: [] }, +}; + +test("counts retries separately without interpreting finding prose", async () => { + const f = fixture([failed, passed, { id: "pending", result: null }]); + const result = await reviewEvidenceReport([f.record], async () => f.bytes); + expect(result.totals).toEqual({ + attempts: 1, + reviewObserved: 1, + noReviewObserved: 0, + unassessed: 0, + passed: 1, + failed: 1, + pending: 1, + codeDefects: 0, + evidenceGaps: 0, + unclassifiedFindings: 1, + }); +}); + +test("separates missing session/transcript evidence from observed absence", async () => { + const empty = fixture(); + expect( + (await reviewEvidenceReport([empty.record], async () => empty.bytes)).totals + .noReviewObserved, + ).toBe(1); + expect( + (await reviewEvidenceReport([empty.record], async () => null)).totals + .unassessed, + ).toBe(1); + const absent = fixture([], false); + expect( + (await reviewEvidenceReport([absent.record], async () => absent.bytes)) + .totals.unassessed, + ).toBe(1); +}); + +test("rejects changed bytes and mismatched attempt bindings", async () => { + const f = fixture(); + await expect( + reviewEvidenceReport([f.record], async () => Buffer.from("{}")), + ).rejects.toThrow("digest mismatch"); + await expect( + reviewEvidenceReport( + [{ ...f.record, attemptId: "other" }], + async () => f.bytes, + ), + ).rejects.toThrow("binding mismatch"); +}); + +test("requires unique evidence-bound assessments and retains attribution", async () => { + const f = fixture([failed, passed]); + const label = { + attemptId: "attempt", + transcriptSha256: f.record.transcript.sha256, + assignmentId: "first", + findingId: "f1", + category: "evidence-gap", + assessor: "test analyst", + rationale: "Packet lacked required metadata", + }; + const result = await reviewEvidenceReport([f.record], async () => f.bytes, [ + label, + ]); + expect(result.totals.evidenceGaps).toBe(1); + expect(JSON.stringify(result)).toContain("test analyst"); + await expect( + reviewEvidenceReport([f.record], async () => f.bytes, [label, label]), + ).rejects.toThrow("Duplicate"); + await expect( + reviewEvidenceReport([f.record], async () => f.bytes, [ + { ...label, transcriptSha256: `sha256:${"0".repeat(64)}` }, + ]), + ).rejects.toThrow("Stale"); + await expect( + reviewEvidenceReport([f.record], async () => f.bytes, [ + { ...label, findingId: "absent" }, + ]), + ).rejects.toThrow("does not match"); + const defect = await reviewEvidenceReport([f.record], async () => f.bytes, [ + { ...label, category: "code-defect" }, + ]); + expect(defect.totals.codeDefects).toBe(1); +}); + +test("does not count malformed or duplicated review records as clean evidence", async () => { + const invalid = fixture([{ id: "missing-result" }]); + expect( + (await reviewEvidenceReport([invalid.record], async () => invalid.bytes)) + .totals.unassessed, + ).toBe(1); + const duplicate = fixture([passed, passed]); + await expect( + reviewEvidenceReport([duplicate.record], async () => duplicate.bytes), + ).rejects.toThrow("Duplicate review"); +}); + +test("rejects repeated attempts and repeated finding identities", async () => { + const f = fixture([failed]); + await expect( + reviewEvidenceReport([f.record, f.record], async () => f.bytes), + ).rejects.toThrow("Duplicate attempt"); + const duplicated = fixture([ + { ...failed, result: { ...failed.result, findings: [finding, finding] } }, + ]); + await expect( + reviewEvidenceReport([duplicated.record], async () => duplicated.bytes), + ).rejects.toThrow("Duplicate finding"); +}); + +test("counts advisories without evidence but keeps unsupported blockers unassessed", async () => { + const advisory = { + findingId: "advisory", + severity: "advisory", + summary: "Consider documenting the default", + }; + const f = fixture([ + { ...passed, result: { ...passed.result, findings: [advisory] } }, + ]); + const result = await reviewEvidenceReport([f.record], async () => f.bytes); + expect(result.totals).toMatchObject({ + reviewObserved: 1, + passed: 1, + unclassifiedFindings: 1, + unassessed: 0, + }); + const observation = result.attempts[0]; + expect( + observation && "reviews" in observation + ? observation.reviews?.[0]?.findings[0] + : null, + ).toMatchObject(advisory); + for (const evidence of [undefined, " "]) { + const blocked = fixture([ + { + ...failed, + result: { + ...failed.result, + findings: [{ ...advisory, severity: "blocking", evidence }], + }, + }, + ]); + expect( + (await reviewEvidenceReport([blocked.record], async () => blocked.bytes)) + .totals, + ).toMatchObject({ unassessed: 1, reviewObserved: 0, failed: 0 }); + } +});