feat(evals): report review coverage and assessed findings

.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/finding-assessments.json

@@ -0,0 +1,11 @@
+[
+  {
+    "attemptId": "attempt-cell-a369e1d7d282a7988c9139d3b396342df350ed092d40ee8ad3589258e76aa832",
+    "transcriptSha256": "sha256:27e16438a9a3389542f99ce92fc564e2114b42642409c555113bfc781744362b",
+    "assignmentId": "id_f095b06b4ddea224",
+    "findingId": "reserved-windows-device-filenames.R5-01",
+    "category": "evidence-gap",
+    "assessor": "Codex retained-evidence audit (assistant assessment)",
+    "rationale": "The first packet omitted the baseline inventory required by flow-review. The retry supplied it with the same source digest and passed; this is an evidence handoff gap, not a demonstrated code defect."
+  }
+]

.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/reporting.md

@@ -0,0 +1,73 @@
+# Report retained review observations
+
+Run from the repository root:
+
+```sh
+bun evals/review-report.ts CAMPAIGN_DIRECTORY [ASSESSMENTS.json]
+```
+
+The command reads the campaign's report, catalog, and referenced transcripts and
+prints JSON to stdout. Redirect stdout to save a separate report. It does not
+launch OpenCode, call providers, modify evidence, regrade, or publish anything.
+No runtime or qualification schema changes are required.
+
+## Read the counts
+
+- `reviewObserved`: attempts with at least one retained review assignment.
+- `noReviewObserved`: readable session/archive evidence contains no assignment.
+  This is not proof that no activity occurred outside that evidence.
+- `unassessed`: a transcript is missing, uses an unsupported shape, or lacks
+  readable session/archive evidence. Never treat these attempts as clean reviews.
+- `passed`, `failed`, and `pending`: recorded assignment outcomes, including retry
+  occurrences. Per-assignment terminal dispositions remain visible; a recorded
+  failed result can represent observed-but-unsubmitted work.
+- `codeDefects` and `evidenceGaps`: finding occurrences explicitly classified by
+  an assessor. These are attributed assessments, not verified truth.
+- `unclassifiedFindings`: findings without an assessment, regardless of wording.
+
+The three attempt counts partition attempted runs. They do not include scheduled
+cells that never produced an attempt. Review verdicts count assignments, not
+independent tasks. A repaired retry does not erase the earlier finding.
+
+## Supply an assessment
+
+Use a JSON array with one object per assessed finding. Each object contains
+`attemptId`, `transcriptSha256`, `assignmentId`, `findingId`, `category` (either
+`code-defect` or `evidence-gap`), `assessor`, and `rationale`. Copy identities from
+the unclassified output after inspecting the evidence. Omit uncertain findings.
+An assessment with a changed digest, unmatched identity, or duplicate key fails.
+The command does not establish an assessor's identity or human-label provenance.
+Never use this file as calibration or promotion evidence.
+
+[finding-assessments.json](finding-assessments.json) supplies the assistant's
+assessment of the one inventory finding in the retained 8.3.0 campaign. It is
+separate from the original transcripts and release report.
+
+```sh
+bun evals/review-report.ts \
+  .release-artifacts/8.3.0-linux/evidence/2026-09-13T15-40-34-840Z.v2 \
+  .agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/finding-assessments.json
+```
+
+The observed result is 76 attempts: 23 with review, 53 with no review observed,
+and none unassessed. There are 23 passed verdicts and one failed verdict, with
+one assessed evidence gap. Without the assessment file that finding remains
+unclassified. The 76/76 release conformance result is untouched.
+
+## Verification and boundaries
+
+Offline tests exercise missing evidence, unsupported session shapes, retries,
+pending assignments, changed transcript bytes, swapped attempt identities,
+duplicate records, and stale/unmatched/duplicate assessments. The actual retained
+campaign was rendered with and without the assessment file, reproducing the
+earlier audit counts without a model call.
+
+The reader validates report/catalog semantics, transcript hashes and attempt
+bindings, and rejects transcript symlinks escaping the campaign directory.
+Retained sessions are redacted; live session operation-digest invariants therefore
+do not apply. A dedicated observation schema validates the fields counted here.
+Hash agreement proves consistency with the supplied report, not independent
+authenticity. Unsupported transcript formats remain unassessed.
+
+This implements audit recommendation 2. Reviewer accuracy, human calibration,
+prompt efficiency, and qualification decisions remain outside this report.

docs/release-qualification.md

@@ -115,3 +115,7 @@ ordinary OpenCode on hidden-graded tasks. It is not a qualification input.
 Scheduled campaigns require explicit authorization and a dispatch budget.
 They retain the ledger and complete campaign directory. Sealing still requires
 the exact-artifact canary. Workflow retries never start another paid campaign.
+
+## Retained review reporting
+
+For offline review counts and assessed findings, see [reporting](../.agents/plans/04-model-adaptive-overhaul/evidence/f6-retained/reporting.md).

evals/review-report.ts

@@ -0,0 +1,266 @@
+import { createHash } from "node:crypto";
+import { readFile, realpath } from "node:fs/promises";
+import { isAbsolute, relative, resolve } from "node:path";
+import { z } from "zod";
+import { parseCaseCatalog } from "./catalog.js";
+import { RetainedScenarioEvidenceSchema } from "./grader-input.js";
+import { type AttemptRecordV2, parseReport } from "./report.js";
+
+const Text = z.string().trim().min(1);
+// Retained transcripts redact operation payloads, so live-session invariants
+// cannot validate them. Validate only the observations used by this report.
+const ReviewDocumentSchema = z.object({
+	runs: z.array(
+		z.object({
+			reviews: z.array(
+				z.object({
+					id: Text,
+					result: z
+						.object({
+							verdict: z.enum(["passed", "failed"]),
+							terminalDisposition: Text.optional(),
+							findings: z.array(
+								z
+									.object({
+										findingId: Text.optional(),
+										severity: z.enum(["blocking", "advisory"]),
+										summary: Text,
+										evidence: Text.optional(),
+									})
+									.refine(
+										(finding) =>
+											finding.severity !== "blocking" ||
+											Boolean(finding.evidence),
+										{
+											message: "A blocking finding requires concrete evidence.",
+											path: ["evidence"],
+										},
+									),
+							),
+						})
+						.nullable(),
+				}),
+			),
+		}),
+	),
+});
+export const FindingAssessmentsSchema = z.array(
+	z
+		.object({
+			attemptId: Text,
+			transcriptSha256: z.string().regex(/^sha256:[a-f0-9]{64}$/),
+			assignmentId: Text,
+			findingId: Text,
+			category: z.enum(["code-defect", "evidence-gap"]),
+			assessor: Text,
+			rationale: Text,
+		})
+		.strict(),
+);
+type Assessments = z.infer<typeof FindingAssessmentsSchema>;
+
+/** Descriptive observations only. Never supplies grades or calibration labels. */
+export async function reviewEvidenceReport(
+	records: readonly Pick<
+		AttemptRecordV2,
+		"attemptId" | "caseId" | "cellId" | "repetition" | "transcript"
+	>[],
+	load: (artifact: string) => Promise<Uint8Array | null>,
+	assessmentInput: unknown = [],
+) {
+	if (
+		new Set(records.map((record) => record.attemptId)).size !== records.length
+	)
+		throw new Error("Duplicate attempt.");
+	const assessments = FindingAssessmentsSchema.parse(assessmentInput);
+	const key = (
+		value: Pick<
+			Assessments[number],
+			"attemptId" | "assignmentId" | "findingId"
+		>,
+	) => JSON.stringify([value.attemptId, value.assignmentId, value.findingId]);
+	const labels = new Map(assessments.map((item) => [key(item), item]));
+	if (labels.size !== assessments.length)
+		throw new Error("Duplicate finding assessment.");
+	const used = new Set<string>();
+	const totals = {
+		attempts: records.length,
+		reviewObserved: 0,
+		noReviewObserved: 0,
+		unassessed: 0,
+		passed: 0,
+		failed: 0,
+		pending: 0,
+		codeDefects: 0,
+		evidenceGaps: 0,
+		unclassifiedFindings: 0,
+	};
+	const attempts = [];
+	for (const attempt of records) {
+		const base = { attemptId: attempt.attemptId, caseId: attempt.caseId };
+		const bytes = attempt.transcript
+			? await load(attempt.transcript.artifact)
+			: null;
+		if (!bytes || !attempt.transcript) {
+			totals.unassessed++;
+			attempts.push({
+				...base,
+				status: "unassessed",
+				reason: "Transcript unavailable",
+			});
+			continue;
+		}
+		const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+		if (digest !== attempt.transcript.sha256)
+			throw new Error("Transcript digest mismatch.");
+		const input: unknown = JSON.parse(Buffer.from(bytes).toString("utf8"));
+		const parsed = RetainedScenarioEvidenceSchema.safeParse(input);
+		if (!parsed.success) {
+			totals.unassessed++;
+			attempts.push({
+				...base,
+				status: "unassessed",
+				reason: "Unsupported retained transcript shape",
+			});
+			continue;
+		}
+		const retained = parsed.data;
+		if (
+			retained.attempt.attemptId !== attempt.attemptId ||
+			retained.attempt.cellId !== attempt.cellId ||
+			retained.attempt.caseId !== attempt.caseId ||
+			retained.attempt.repetition !== attempt.repetition
+		) {
+			throw new Error("Transcript attempt binding mismatch.");
+		}
+		const documents = [...retained.gradeInput.archives];
+		if (retained.gradeInput.session)
+			documents.push(retained.gradeInput.session);
+		const sessions = documents.map((document) =>
+			ReviewDocumentSchema.safeParse(document),
+		);
+		if (sessions.length === 0 || sessions.some((session) => !session.success)) {
+			totals.unassessed++;
+			attempts.push({
+				...base,
+				status: "unassessed",
+				reason: "Session evidence unavailable or unsupported",
+			});
+			continue;
+		}
+		const reviews = sessions.flatMap((session) =>
+			session.success ? session.data.runs.flatMap((run) => run.reviews) : [],
+		);
+		if (new Set(reviews.map((review) => review.id)).size !== reviews.length)
+			throw new Error("Duplicate review assignment.");
+		if (reviews.length) totals.reviewObserved++;
+		else totals.noReviewObserved++;
+		const observations = reviews.map((review) => {
+			const ids = (review.result?.findings ?? []).flatMap((finding) =>
+				finding.findingId ? [finding.findingId] : [],
+			);
+			if (new Set(ids).size !== ids.length)
+				throw new Error("Duplicate finding identity.");
+			if (review.result) totals[review.result.verdict]++;
+			else totals.pending++;
+			return {
+				assignmentId: review.id,
+				verdict: review.result?.verdict ?? "pending",
+				terminalDisposition: review.result?.terminalDisposition ?? null,
+				findings: (review.result?.findings ?? []).map((finding) => {
+					const id = key({
+						...base,
+						assignmentId: review.id,
+						findingId: finding.findingId ?? "",
+					});
+					const label = labels.get(id);
+					if (label && label.transcriptSha256 !== digest)
+						throw new Error("Stale finding assessment.");
+					if (label) used.add(id);
+					if (label?.category === "code-defect") totals.codeDefects++;
+					else if (label?.category === "evidence-gap") totals.evidenceGaps++;
+					else totals.unclassifiedFindings++;
+					return {
+						...finding,
+						category: label?.category ?? "unclassified",
+						assessment: label ?? null,
+					};
+				}),
+			};
+		});
+		attempts.push({
+			...base,
+			transcriptSha256: digest,
+			status: reviews.length ? "review-observed" : "no-review-observed",
+			reviews: observations,
+		});
+	}
+	if (used.size !== labels.size)
+		throw new Error("Assessment does not match an observed finding.");
+	return {
+		kind: "descriptive-review-evidence",
+		limitations: [
+			"No review observed is not proof that no review activity occurred outside retained session evidence.",
+			"Verdicts and finding occurrences include retries; they are not independent tasks or accuracy estimates.",
+			"Categories are attributed assessments, not verified defects, human calibration, or release grades.",
+		],
+		totals,
+		attempts,
+	};
+}
+
+export async function readReviewEvidenceReport(
+	directory: string,
+	assessments: unknown = [],
+) {
+	const root = await realpath(directory);
+	const catalog = parseCaseCatalog(
+		JSON.parse(await readFile(resolve(root, "catalog.json"), "utf8")),
+	);
+	if (!catalog.ok) throw new Error("Invalid case catalog.");
+	const report = parseReport(
+		JSON.parse(await readFile(resolve(root, "report.json"), "utf8")),
+		catalog.value,
+	);
+	if (!report.ok) throw new Error("Invalid campaign report.");
+	return reviewEvidenceReport(
+		report.value.attempts,
+		async (artifact) => {
+			let path: string;
+			try {
+				path = await realpath(resolve(root, artifact));
+			} catch (error) {
+				if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+				throw error;
+			}
+			const local = relative(root, path);
+			if (
+				isAbsolute(local) ||
+				local === ".." ||
+				local.startsWith("../") ||
+				local.startsWith("..\\")
+			)
+				throw new Error("Transcript escapes campaign directory.");
+			return readFile(path);
+		},
+		assessments,
+	);
+}
+
+if (import.meta.main) {
+	const [directory, assessmentsPath, ...extra] = process.argv.slice(2);
+	if (!directory || extra.length)
+		throw new Error(
+			"Usage: bun evals/review-report.ts CAMPAIGN [ASSESSMENTS.json]",
+		);
+	const assessments: unknown = assessmentsPath
+		? JSON.parse(await readFile(assessmentsPath, "utf8"))
+		: [];
+	console.log(
+		JSON.stringify(
+			await readReviewEvidenceReport(directory, assessments),
+			null,
+			2,
+		),
+	);
+}