From 1e0d2770f05d2d508b5f08a073c9e69d4dcd8a90 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 19:19:11 +0000 Subject: [PATCH 1/7] records(export): structured record exports and the export format Deliver Milestone 5 slice 1 (#45; #44 decision 1): finalized versions leave the installation as verifiable archives whose record bytes are the stored canonical bytes, byte for byte. - record_export: units (stored bytes beside a canonical-JSON unit manifest), ZIP archives with stored entries in a fixed order and metadata (deterministic per scope and instant), four scopes (version, record, enrollment, installation), read-rule authorization with export_records for the installation, typed refusals including nothing_to_export, record_exported audit events, and verification from the archive alone with typed findings and predecessor linking. - exports_http: downloads for every scope and the export summary; main: `export verify ` opens no data directory. - web: export buttons on the record page (this version, all versions), the enrollment page, and an installation-export card on Home; downloads surface refusals as errors instead of saved error bodies. - docs: docs/formats/record-export.md (normative), ADR 0014, the RecordExport domain-model entry; zip added without default features. - tests: verbatim bytes, determinism, every verification finding, scope authorization, API delivery, CLI verification; the drafts e2e downloads a version export from the browser and verifies it with the CLI. Refs #44 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- Cargo.lock | 28 + crates/consolebook-server/Cargo.toml | 1 + crates/consolebook-server/src/audit.rs | 2 + crates/consolebook-server/src/exports_http.rs | 122 ++ crates/consolebook-server/src/http.rs | 1 + crates/consolebook-server/src/lib.rs | 2 + crates/consolebook-server/src/main.rs | 65 +- .../consolebook-server/src/record_export.rs | 976 ++++++++++++++ .../consolebook-server/tests/record_export.rs | 1163 +++++++++++++++++ docs/decisions/0014-record-export-format.md | 140 ++ docs/domain-model.md | 6 + docs/formats/record-export.md | 243 ++++ web/e2e/drafts.spec.ts | 13 + web/src/lib/api.ts | 65 + web/src/routes/+page.svelte | 62 + web/src/routes/drafts/[id]/+page.svelte | 55 + web/src/routes/enrollments/[id]/+page.svelte | 40 + 17 files changed, 2983 insertions(+), 1 deletion(-) create mode 100644 crates/consolebook-server/src/exports_http.rs create mode 100644 crates/consolebook-server/src/record_export.rs create mode 100644 crates/consolebook-server/tests/record_export.rs create mode 100644 docs/decisions/0014-record-export-format.md create mode 100644 docs/formats/record-export.md diff --git a/Cargo.lock b/Cargo.lock index e1eeaaf..d0697d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -340,6 +340,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "zip", ] [[package]] @@ -398,6 +399,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" version = "0.3.13" @@ -2281,6 +2291,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -2664,6 +2680,18 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/consolebook-server/Cargo.toml b/crates/consolebook-server/Cargo.toml index 0f6773c..5cfbfa5 100644 --- a/crates/consolebook-server/Cargo.toml +++ b/crates/consolebook-server/Cargo.toml @@ -33,6 +33,7 @@ uuid = { version = "1", features = ["v4"] } # so local-time resolution has the one-executable semantics ADR 0009 # documents, on every platform. jiff = { version = "0.2", default-features = false, features = ["std", "tzdb-bundle-always"] } +zip = { version = "8.6.0", default-features = false } [dev-dependencies] http-body-util = "0.1" diff --git a/crates/consolebook-server/src/audit.rs b/crates/consolebook-server/src/audit.rs index be17e17..866d1f5 100644 --- a/crates/consolebook-server/src/audit.rs +++ b/crates/consolebook-server/src/audit.rs @@ -48,6 +48,7 @@ pub enum EventKind { AcknowledgmentRecorded, AmendmentOpened, TaskSignoffRecorded, + RecordExported, } impl EventKind { @@ -90,6 +91,7 @@ impl EventKind { Self::AcknowledgmentRecorded => "acknowledgment_recorded", Self::AmendmentOpened => "amendment_opened", Self::TaskSignoffRecorded => "task_signoff_recorded", + Self::RecordExported => "record_exported", } } } diff --git a/crates/consolebook-server/src/exports_http.rs b/crates/consolebook-server/src/exports_http.rs new file mode 100644 index 0000000..485e1f1 --- /dev/null +++ b/crates/consolebook-server/src/exports_http.rs @@ -0,0 +1,122 @@ +//! Record export HTTP handlers (Milestone 5 slice 1; ADR 0014). +//! +//! `http.rs` remains the hub; this module owns the export downloads and +//! the installation-export summary. Scope rules live in +//! `record_export`; handlers translate refusals into stable error codes +//! and deliver the documented archive bytes as attachments. + +use axum::Router; +use axum::extract::{Path, State}; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Json, Response}; +use axum::routing::get; + +use crate::http::{ApiError, AppState, CurrentUser}; +use crate::record_export::{self, ExportRefusal, Scope}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/api/drafts/{id}/export", get(export_record)) + .route( + "/api/drafts/{id}/versions/{number}/export", + get(export_version), + ) + .route("/api/enrollments/{id}/export", get(export_enrollment)) + .route("/api/exports/records", get(export_installation)) + .route("/api/exports/summary", get(export_summary)) +} + +fn export_refusal(refusal: ExportRefusal) -> ApiError { + match refusal { + ExportRefusal::NoSuchRecord => { + ApiError::new(StatusCode::NOT_FOUND, "no_such_record", "no such record") + } + ExportRefusal::NoSuchVersion => ApiError::new( + StatusCode::NOT_FOUND, + "no_such_version", + "this record has no finalized version with that number", + ), + ExportRefusal::NoSuchEnrollment => ApiError::new( + StatusCode::NOT_FOUND, + "no_such_enrollment", + "no such enrollment", + ), + ExportRefusal::CapabilityRequired => ApiError::new( + StatusCode::FORBIDDEN, + "capability_required", + "exporting takes the scope's read authority; the whole installation takes export_records", + ), + ExportRefusal::NothingToExport => ApiError::new( + StatusCode::CONFLICT, + "nothing_to_export", + "this scope holds no finalized version; an export never claims completeness it lacks", + ), + } +} + +/// The archive as a download: exactly the documented bytes. +async fn deliver(state: &AppState, actor_user_id: i64, scope: Scope) -> Result { + match record_export::export(&state.pool, actor_user_id, scope).await? { + Ok(export) => { + let disposition = format!("attachment; filename=\"{}\"", export.file_name); + Ok(( + [ + (header::CONTENT_TYPE, "application/zip".to_owned()), + (header::CONTENT_DISPOSITION, disposition), + ], + export.bytes, + ) + .into_response()) + } + Err(refusal) => Err(export_refusal(refusal)), + } +} + +async fn export_version( + State(state): State, + current: CurrentUser, + Path((record_id, version_number)): Path<(i64, i64)>, +) -> Result { + deliver( + &state, + current.user.id, + Scope::Version { + record_id, + version_number, + }, + ) + .await +} + +async fn export_record( + State(state): State, + current: CurrentUser, + Path(record_id): Path, +) -> Result { + deliver(&state, current.user.id, Scope::Record { record_id }).await +} + +async fn export_enrollment( + State(state): State, + current: CurrentUser, + Path(enrollment_id): Path, +) -> Result { + deliver(&state, current.user.id, Scope::Enrollment { enrollment_id }).await +} + +async fn export_installation( + State(state): State, + current: CurrentUser, +) -> Result { + deliver(&state, current.user.id, Scope::Installation).await +} + +async fn export_summary( + State(state): State, + current: CurrentUser, +) -> Result { + match record_export::summary(&state.pool, current.user.id).await? { + Ok(summary) => Ok(Json(summary).into_response()), + Err(refusal) => Err(export_refusal(refusal)), + } +} diff --git a/crates/consolebook-server/src/http.rs b/crates/consolebook-server/src/http.rs index d3d54b5..66da178 100644 --- a/crates/consolebook-server/src/http.rs +++ b/crates/consolebook-server/src/http.rs @@ -55,6 +55,7 @@ pub fn router(state: AppState) -> Router { .merge(crate::programs_http::routes()) .merge(crate::training_http::routes()) .merge(crate::drafts_http::routes()) + .merge(crate::exports_http::routes()) .fallback(crate::web_assets::serve) .with_state(state) } diff --git a/crates/consolebook-server/src/lib.rs b/crates/consolebook-server/src/lib.rs index 7777dd0..7fc50a8 100644 --- a/crates/consolebook-server/src/lib.rs +++ b/crates/consolebook-server/src/lib.rs @@ -18,6 +18,7 @@ pub mod draft_review; pub mod drafts_http; pub mod enrollments; pub mod evaluation_drafts; +pub mod exports_http; pub mod finalization; pub mod http; pub mod lifecycle; @@ -25,6 +26,7 @@ pub mod notices; pub mod program_export; pub mod programs; pub mod programs_http; +pub mod record_export; pub mod restore; pub mod scheduler; pub mod secrets; diff --git a/crates/consolebook-server/src/main.rs b/crates/consolebook-server/src/main.rs index a9c97cf..c23edf3 100644 --- a/crates/consolebook-server/src/main.rs +++ b/crates/consolebook-server/src/main.rs @@ -12,7 +12,7 @@ use consolebook_server::doctor::Verdict; use consolebook_server::serve_lock::ServeLock; use consolebook_server::users::{IssueRefusal, ResetOrigin}; use consolebook_server::{ - VERSION, backup, doctor, http, restore, scheduler, setup, storage, users, + VERSION, backup, doctor, http, record_export, restore, scheduler, setup, storage, users, }; #[derive(Parser)] @@ -77,6 +77,11 @@ enum Command { /// Path to the snapshot to restore, usually in backups/. snapshot: PathBuf, }, + /// Work with record exports (docs/formats/record-export.md). + Export { + #[command(subcommand)] + action: ExportAction, + }, /// Print a fresh first-run setup code for an uninitialized installation. SetupCode, /// Issue a password reset code for a locked-out administrator. @@ -90,6 +95,19 @@ enum Command { }, } +#[derive(Subcommand)] +enum ExportAction { + /// Verify a record export archive from the file alone. + /// + /// Opens no data directory: the archive carries everything the + /// checks need, and the verdict is consistency with its stated + /// fingerprints. + Verify { + /// Path to the export archive (.zip). + archive: PathBuf, + }, +} + fn init_logging() { use tracing_subscriber::EnvFilter; // Structured logs, no record content: log events describe operations and @@ -114,6 +132,9 @@ async fn main() -> ExitCode { Command::Doctor => run_doctor(&cli.data_dir).await, Command::Backup { keep } => run_backup(&cli.data_dir, keep).await, Command::Restore { snapshot } => run_restore(&cli.data_dir, &snapshot).await, + Command::Export { + action: ExportAction::Verify { archive }, + } => run_export_verify(&archive), Command::SetupCode => run_setup_code(&cli.data_dir).await, Command::Recover { username } => run_recover(&cli.data_dir, &username).await, }; @@ -274,3 +295,45 @@ async fn run_restore(data_dir: &std::path::Path, snapshot: &std::path::Path) -> ); Ok(ExitCode::SUCCESS) } + +fn run_export_verify(archive: &std::path::Path) -> Result { + let bytes = std::fs::read(archive).with_context(|| format!("reading {}", archive.display()))?; + let report = record_export::verify_archive(&bytes); + if let Some(id) = &report.installation_id { + println!("installation {id}"); + } + if let Some(at) = report.exported_at_rfc3339() { + println!("exported at {at}"); + } + if let Some(scope) = report.scope { + println!("scope {scope}"); + } + for finding in &report.findings { + println!("FAIL archive {finding}"); + } + for unit in &report.units { + let mark = if unit.verified() { "ok " } else { "FAIL" }; + println!( + "{mark} {:<20} record {} version {} (schema {}); predecessor {}", + unit.path, unit.record_id, unit.version_number, unit.record_schema, unit.predecessor + ); + for finding in &unit.findings { + println!(" {finding}"); + } + } + let consistent = report.units.iter().filter(|unit| unit.verified()).count(); + if report.verified() { + println!( + "verified {consistent} of {} units: the export is consistent with its stated fingerprints", + report.units.len() + ); + Ok(ExitCode::SUCCESS) + } else { + println!( + "NOT VERIFIED: {consistent} of {} units consistent, {} archive finding(s)", + report.units.len(), + report.findings.len() + ); + Ok(ExitCode::FAILURE) + } +} diff --git a/crates/consolebook-server/src/record_export.rs b/crates/consolebook-server/src/record_export.rs new file mode 100644 index 0000000..8a25404 --- /dev/null +++ b/crates/consolebook-server/src/record_export.rs @@ -0,0 +1,976 @@ +//! Structured record exports (ADR 0014; docs/formats/record-export.md; +//! #45). +//! +//! An export unit is one finalized version's stored canonical bytes, +//! copied verbatim, beside a canonical-JSON unit manifest; an archive +//! is a ZIP container with stored entries in a fixed order holding an +//! archive manifest and its units. The archive is a pure function of +//! the scope's stored rows and the export instant, so the same scope +//! exported at the same instant is byte-identical. Verification reads +//! only the archive and reports typed findings; its verdict is +//! consistency with the stated fingerprints, never tamper-proofing +//! (ADR 0010, ADR 0011). Export follows the read rules that already +//! exist — a unit contains exactly what its reader may already read — +//! and the installation scope takes `export_records`. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::io::{Cursor, Read, Write}; + +use anyhow::{Context, Result, anyhow}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::{Row, SqlitePool}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use zip::CompressionMethod; +use zip::write::{SimpleFileOptions, ZipWriter}; + +use crate::audit::{self, EventKind, Subject}; +use crate::canonical; +use crate::capabilities::{self, Capability}; +use crate::evaluation_drafts; +use crate::lifecycle; +use crate::storage; + +/// Archive-manifest discriminator; never changes. +pub const ARCHIVE_FORMAT: &str = "consolebook-record-export"; +/// Unit-manifest discriminator; never changes. +pub const UNIT_FORMAT: &str = "consolebook-record-unit"; +/// Shared by both manifests; bumped by any change to either shape. +pub const FORMAT_VERSION: i64 = 1; +/// The archive manifest's entry name. +pub const ARCHIVE_MANIFEST_PATH: &str = "manifest.json"; +/// The canonical record bytes within a unit directory. +pub const RECORD_FILE: &str = "record.json"; +/// The unit manifest within a unit directory. +pub const UNIT_MANIFEST_FILE: &str = "manifest.json"; + +/// What an archive claims to contain. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Scope { + /// Exactly one finalized version. + Version { record_id: i64, version_number: i64 }, + /// Every retained version of one record, superseded originals + /// included. + Record { record_id: i64 }, + /// Every finalized version of every record of one enrollment. + Enrollment { enrollment_id: i64 }, + /// Every finalized version the installation holds. + Installation, +} + +impl fmt::Display for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Version { + record_id, + version_number, + } => write!(f, "record {record_id}, version {version_number}"), + Self::Record { record_id } => { + write!(f, "record {record_id}, every retained version") + } + Self::Enrollment { enrollment_id } => write!(f, "enrollment {enrollment_id}"), + Self::Installation => f.write_str("the whole installation"), + } + } +} + +/// Typed refusals for the export act. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExportRefusal { + NoSuchRecord, + NoSuchVersion, + NoSuchEnrollment, + CapabilityRequired, + /// The scope exists but holds no finalized version; an empty + /// archive is never presented as a complete export. + NothingToExport, +} + +/// One unit as the archive manifest lists it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnitEntry { + pub path: String, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub content_hash: String, + pub chain_hash: String, + pub predecessor_content_hash: Option, +} + +/// The archive manifest (`manifest.json` at the root). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArchiveManifest { + pub format: String, + pub format_version: i64, + pub installation_id: String, + pub exported_at: i64, + pub scope: Scope, + pub units: Vec, +} + +/// The unit manifest beside each unit's record bytes. It repeats what +/// the archive manifest says so a unit directory stands on its own. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnitManifest { + pub format: String, + pub format_version: i64, + pub installation_id: String, + pub exported_at: i64, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub content_hash: String, + pub chain_hash: String, + pub predecessor_content_hash: Option, +} + +/// A produced archive, ready to deliver. +#[derive(Debug)] +pub struct Export { + /// The documented download name, `consolebook--.zip`. + pub file_name: String, + pub bytes: Vec, + pub exported_at: i64, + pub unit_count: usize, +} + +/// The unit directory for one version: `records/{record_id}/v{n}`. +#[must_use] +pub fn unit_path(record_id: i64, version_number: i64) -> String { + format!("records/{record_id}/v{version_number}") +} + +// ------------------------------------------------------------ producing + +/// Exports `scope` now, for an actor the scope's read rule admits. +pub async fn export( + pool: &SqlitePool, + actor_user_id: i64, + scope: Scope, +) -> Result> { + export_at( + pool, + actor_user_id, + scope, + OffsetDateTime::now_utc().unix_timestamp(), + ) + .await +} + +/// Exports `scope` stamped with `exported_at` (UTC unix seconds). The +/// archive is a pure function of the scope's rows and this instant. +pub async fn export_at( + pool: &SqlitePool, + actor_user_id: i64, + scope: Scope, + exported_at: i64, +) -> Result> { + let audited = match authorize(pool, actor_user_id, scope).await? { + Ok(audited) => audited, + Err(refusal) => return Ok(Err(refusal)), + }; + let rows = collect(pool, scope).await?; + if rows.is_empty() { + return Ok(Err(match scope { + Scope::Version { .. } => ExportRefusal::NoSuchVersion, + Scope::Record { .. } | Scope::Enrollment { .. } | Scope::Installation => { + ExportRefusal::NothingToExport + } + })); + } + let installation_id = storage::installation_id(pool).await?; + let bytes = build_archive(&installation_id, exported_at, scope, &rows)?; + // The export is audited once it exists: actor and subject, never + // content (docs/records-integrity.md). + match audited.subject { + Some(subject) => { + audit::record_for_subject( + pool, + EventKind::RecordExported, + Some(actor_user_id), + audited.trainee, + subject, + ) + .await?; + } + None => audit::record(pool, EventKind::RecordExported, Some(actor_user_id), None).await?, + } + Ok(Ok(Export { + file_name: file_name(scope, exported_at)?, + bytes, + exported_at, + unit_count: rows.len(), + })) +} + +/// Counts for the installation-export interface, for `export_records` +/// holders. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExportSummary { + pub installation_id: String, + pub record_count: i64, + pub version_count: i64, +} + +/// How many finalized records and versions an installation export +/// would carry. +pub async fn summary( + pool: &SqlitePool, + actor_user_id: i64, +) -> Result> { + if !capabilities::user_has(pool, actor_user_id, Capability::ExportRecords).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + let (record_count, version_count): (i64, i64) = sqlx::query_as( + "SELECT COUNT(DISTINCT evaluation_record_id), COUNT(*) FROM evaluation_version", + ) + .fetch_one(pool) + .await + .context("counting finalized versions")?; + Ok(Ok(ExportSummary { + installation_id: storage::installation_id(pool).await?, + record_count, + version_count, + })) +} + +/// What the audit event names once the export exists. +struct Audited { + subject: Option, + trainee: Option, +} + +/// The scope's read rule, as the typed contract it already is elsewhere +/// (ADR 0010): the record read rule for a version or record, the +/// training-history read rule for an enrollment, and the explicit +/// `export_records` authority for the installation. +async fn authorize( + pool: &SqlitePool, + actor_user_id: i64, + scope: Scope, +) -> Result> { + match scope { + Scope::Version { record_id, .. } | Scope::Record { record_id } => { + let mut conn = pool.acquire().await.context("acquiring connection")?; + let Some(record) = evaluation_drafts::load_record(&mut conn, record_id).await? else { + return Ok(Err(ExportRefusal::NoSuchRecord)); + }; + drop(conn); + if !crate::draft_access::may_read(pool, actor_user_id, &record).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + let trainee: i64 = sqlx::query_scalar("SELECT user_id FROM enrollment WHERE id = ?1") + .bind(record.enrollment_id) + .fetch_one(pool) + .await + .context("reading enrollment")?; + Ok(Ok(Audited { + subject: Some(Subject::Record(record_id)), + trainee: Some(trainee), + })) + } + Scope::Enrollment { enrollment_id } => { + let trainee: Option = + sqlx::query_scalar("SELECT user_id FROM enrollment WHERE id = ?1") + .bind(enrollment_id) + .fetch_optional(pool) + .await + .context("reading enrollment")?; + let Some(trainee) = trainee else { + return Ok(Err(ExportRefusal::NoSuchEnrollment)); + }; + if !lifecycle::may_read(pool, actor_user_id, enrollment_id).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + Ok(Ok(Audited { + subject: Some(Subject::Enrollment(enrollment_id)), + trainee: Some(trainee), + })) + } + Scope::Installation => { + if !capabilities::user_has(pool, actor_user_id, Capability::ExportRecords).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + Ok(Ok(Audited { + subject: None, + trainee: None, + })) + } + } +} + +/// One stored version, exactly as the archive carries it. +struct VersionRow { + record_id: i64, + version_number: i64, + record_schema: i64, + bytes: Vec, + content_hash: String, + chain_hash: String, + predecessor_content_hash: Option, +} + +/// The stored rows of a scope in archive order: ascending record id, +/// then version number. The predecessor's content hash is read from +/// its own row, so the manifest states what the chain hash was +/// computed over (ADR 0011). +macro_rules! unit_query { + ($where:literal) => { + concat!( + "SELECT v.evaluation_record_id AS record_id, v.version_number, + v.record_schema, v.canonical_bytes, v.content_hash, + v.chain_hash, p.content_hash AS predecessor_content_hash + FROM evaluation_version v + LEFT JOIN evaluation_version p ON p.id = v.predecessor_id + JOIN evaluation_record r ON r.id = v.evaluation_record_id ", + $where, + " ORDER BY v.evaluation_record_id, v.version_number" + ) + }; +} + +async fn collect(pool: &SqlitePool, scope: Scope) -> Result> { + let rows = match scope { + Scope::Version { + record_id, + version_number, + } => { + sqlx::query(unit_query!( + "WHERE v.evaluation_record_id = ?1 AND v.version_number = ?2" + )) + .bind(record_id) + .bind(version_number) + .fetch_all(pool) + .await + } + Scope::Record { record_id } => { + sqlx::query(unit_query!("WHERE v.evaluation_record_id = ?1")) + .bind(record_id) + .fetch_all(pool) + .await + } + Scope::Enrollment { enrollment_id } => { + sqlx::query(unit_query!("WHERE r.enrollment_id = ?1")) + .bind(enrollment_id) + .fetch_all(pool) + .await + } + Scope::Installation => sqlx::query(unit_query!("")).fetch_all(pool).await, + } + .context("reading finalized versions")?; + Ok(rows + .iter() + .map(|row| VersionRow { + record_id: row.get("record_id"), + version_number: row.get("version_number"), + record_schema: row.get("record_schema"), + bytes: row.get("canonical_bytes"), + content_hash: row.get("content_hash"), + chain_hash: row.get("chain_hash"), + predecessor_content_hash: row.get("predecessor_content_hash"), + }) + .collect()) +} + +/// Writes the container exactly as docs/formats/record-export.md lays +/// it out: manifest first, then units in order, stored entries, the +/// export instant as every entry's modification time, `0644`. +fn build_archive( + installation_id: &str, + exported_at: i64, + scope: Scope, + rows: &[VersionRow], +) -> Result> { + let units: Vec = rows + .iter() + .map(|row| UnitEntry { + path: unit_path(row.record_id, row.version_number), + record_id: row.record_id, + version_number: row.version_number, + record_schema: row.record_schema, + content_hash: row.content_hash.clone(), + chain_hash: row.chain_hash.clone(), + predecessor_content_hash: row.predecessor_content_hash.clone(), + }) + .collect(); + let manifest = ArchiveManifest { + format: ARCHIVE_FORMAT.to_owned(), + format_version: FORMAT_VERSION, + installation_id: installation_id.to_owned(), + exported_at, + scope, + units, + }; + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .last_modified_time(dos_time(exported_at)?) + .unix_permissions(0o644); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + add_entry( + &mut writer, + ARCHIVE_MANIFEST_PATH, + &canonical_json(&manifest)?, + options, + )?; + for (row, entry) in rows.iter().zip(&manifest.units) { + add_entry( + &mut writer, + &format!("{}/{RECORD_FILE}", entry.path), + &row.bytes, + options, + )?; + let unit = UnitManifest { + format: UNIT_FORMAT.to_owned(), + format_version: FORMAT_VERSION, + installation_id: installation_id.to_owned(), + exported_at, + record_id: entry.record_id, + version_number: entry.version_number, + record_schema: entry.record_schema, + content_hash: entry.content_hash.clone(), + chain_hash: entry.chain_hash.clone(), + predecessor_content_hash: entry.predecessor_content_hash.clone(), + }; + add_entry( + &mut writer, + &format!("{}/{UNIT_MANIFEST_FILE}", entry.path), + &canonical_json(&unit)?, + options, + )?; + } + let cursor = writer.finish().context("finishing the export archive")?; + Ok(cursor.into_inner()) +} + +fn add_entry( + writer: &mut ZipWriter>>, + name: &str, + bytes: &[u8], + options: SimpleFileOptions, +) -> Result<()> { + writer + .start_file(name, options) + .with_context(|| format!("starting archive entry {name}"))?; + writer + .write_all(bytes) + .with_context(|| format!("writing archive entry {name}"))?; + Ok(()) +} + +/// Manifests are canonical JSON under the record format's subset, so +/// the archive is deterministic and a manifest is itself checkable. +fn canonical_json(value: &T) -> Result> { + let value = serde_json::to_value(value).context("serializing manifest")?; + canonical::canonical_bytes(&value) +} + +fn dos_time(exported_at: i64) -> Result { + let at = OffsetDateTime::from_unix_timestamp(exported_at).context("export instant")?; + zip::DateTime::from_date_and_time( + u16::try_from(at.year()).context("export year")?, + u8::from(at.month()), + at.day(), + at.hour(), + at.minute(), + at.second(), + ) + .map_err(|_| anyhow!("export instant {exported_at} is outside the ZIP date range")) +} + +fn file_name(scope: Scope, exported_at: i64) -> Result { + let stamp = OffsetDateTime::from_unix_timestamp(exported_at) + .context("export instant")? + .format(&time::macros::format_description!( + "[year][month][day]T[hour][minute][second]Z" + )) + .context("formatting export instant")?; + let scope_part = match scope { + Scope::Version { + record_id, + version_number, + } => format!("record-{record_id}-v{version_number}"), + Scope::Record { record_id } => format!("record-{record_id}"), + Scope::Enrollment { enrollment_id } => format!("enrollment-{enrollment_id}"), + Scope::Installation => "installation".to_owned(), + }; + Ok(format!("consolebook-{scope_part}-{stamp}.zip")) +} + +// ------------------------------------------------------------ verifying + +/// One thing a verifier found wrong. The verdict derives from the +/// absence of findings; wording is presentation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Finding { + NotAnArchive { + detail: String, + }, + ArchiveManifestMissing, + ArchiveManifestUnreadable { + detail: String, + }, + UnsupportedFormat { + format: String, + format_version: i64, + }, + /// A manifest's bytes are not the canonical serialization of what + /// they parse to: a member is missing, reordered, or reformatted. + ManifestNotCanonical { + path: String, + }, + /// Units are not strictly ascending by (record, version). + UnitsOutOfOrder, + UnitPathUnexpected { + path: String, + expected: String, + }, + /// The container holds an entry the manifest does not name. + UnlistedEntry { + path: String, + }, + MissingEntry { + path: String, + }, + EntryUnreadable { + path: String, + detail: String, + }, + UnitManifestUnreadable { + detail: String, + }, + /// The unit manifest and the archive manifest disagree on a member. + UnitManifestDisagrees { + member: &'static str, + }, + ContentHashMismatch, + /// `record.json` is not canonical bytes (or not JSON at all). + NotCanonical { + detail: String, + }, + /// The envelope's own identity members disagree with the manifest. + EnvelopeDisagrees { + member: &'static str, + }, + ChainHashMismatch, + /// A first version with a predecessor, or a later one without. + LineageShape, + /// The predecessor is in the archive and its content hash is not + /// what this unit's chain was computed over. + PredecessorMismatch, +} + +impl fmt::Display for Finding { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAnArchive { detail } => write!(f, "not a readable ZIP archive: {detail}"), + Self::ArchiveManifestMissing => f.write_str("the archive manifest is missing"), + Self::ArchiveManifestUnreadable { detail } => { + write!(f, "the archive manifest is unreadable: {detail}") + } + Self::UnsupportedFormat { + format, + format_version, + } => write!(f, "unsupported format '{format}' version {format_version}"), + Self::ManifestNotCanonical { path } => { + write!(f, "{path} is not canonical JSON") + } + Self::UnitsOutOfOrder => { + f.write_str("units are not strictly ascending by record and version") + } + Self::UnitPathUnexpected { path, expected } => { + write!(f, "unit path {path} should be {expected}") + } + Self::UnlistedEntry { path } => write!(f, "entry {path} is not listed by the manifest"), + Self::MissingEntry { path } => write!(f, "entry {path} is missing"), + Self::EntryUnreadable { path, detail } => { + write!(f, "entry {path} is unreadable: {detail}") + } + Self::UnitManifestUnreadable { detail } => { + write!(f, "the unit manifest is unreadable: {detail}") + } + Self::UnitManifestDisagrees { member } => { + write!( + f, + "the unit manifest disagrees with the archive on {member}" + ) + } + Self::ContentHashMismatch => { + f.write_str("the content hash does not match the record bytes") + } + Self::NotCanonical { detail } => { + write!(f, "the record bytes are not canonical: {detail}") + } + Self::EnvelopeDisagrees { member } => { + write!(f, "the record's own {member} disagrees with the manifest") + } + Self::ChainHashMismatch => { + f.write_str("the chain hash does not match the predecessor hash and record bytes") + } + Self::LineageShape => { + f.write_str("a predecessor hash is present exactly for versions after the first") + } + Self::PredecessorMismatch => { + f.write_str("the predecessor in this archive has a different content hash") + } + } + } +} + +/// Whether a unit's predecessor was checked against the archive. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PredecessorLink { + /// A first version. + None, + /// The predecessor is in the archive and its content hash matches. + Linked, + /// The archive does not carry the predecessor; the chain hash was + /// still recomputed from the carried predecessor hash. + NotInExport, +} + +impl fmt::Display for PredecessorLink { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::None => "none (first version)", + Self::Linked => "linked", + Self::NotInExport => "not in export", + }) + } +} + +/// One unit's verification. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct UnitReport { + pub path: String, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub predecessor: PredecessorLink, + pub findings: Vec, +} + +impl UnitReport { + #[must_use] + pub fn verified(&self) -> bool { + self.findings.is_empty() + } +} + +/// The whole archive's verification. `verified` when nothing was found +/// wrong anywhere: internally consistent with its stated fingerprints, +/// which is what the format can prove. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ArchiveReport { + pub installation_id: Option, + pub exported_at: Option, + pub scope: Option, + pub units: Vec, + pub findings: Vec, +} + +impl ArchiveReport { + #[must_use] + pub fn verified(&self) -> bool { + self.findings.is_empty() && self.units.iter().all(UnitReport::verified) + } + + /// The export instant as RFC 3339, for presentation. + #[must_use] + pub fn exported_at_rfc3339(&self) -> Option { + self.exported_at.and_then(|at| { + OffsetDateTime::from_unix_timestamp(at) + .ok()? + .format(&Rfc3339) + .ok() + }) + } +} + +type Archive<'a> = zip::ZipArchive>; + +/// Verifies an archive from its bytes alone, per the normative checks +/// in docs/formats/record-export.md. +#[must_use] +pub fn verify_archive(bytes: &[u8]) -> ArchiveReport { + let mut report = ArchiveReport { + installation_id: None, + exported_at: None, + scope: None, + units: Vec::new(), + findings: Vec::new(), + }; + let mut archive = match zip::ZipArchive::new(Cursor::new(bytes)) { + Ok(archive) => archive, + Err(err) => { + report.findings.push(Finding::NotAnArchive { + detail: err.to_string(), + }); + return report; + } + }; + let names: Vec = archive.file_names().map(str::to_owned).collect(); + let manifest_bytes = match read_entry(&mut archive, ARCHIVE_MANIFEST_PATH) { + Ok(Some(bytes)) => bytes, + Ok(None) => { + report.findings.push(Finding::ArchiveManifestMissing); + return report; + } + Err(detail) => { + report.findings.push(Finding::EntryUnreadable { + path: ARCHIVE_MANIFEST_PATH.to_owned(), + detail, + }); + return report; + } + }; + let manifest: ArchiveManifest = match serde_json::from_slice(&manifest_bytes) { + Ok(manifest) => manifest, + Err(err) => { + report.findings.push(Finding::ArchiveManifestUnreadable { + detail: err.to_string(), + }); + return report; + } + }; + if manifest.format != ARCHIVE_FORMAT || manifest.format_version != FORMAT_VERSION { + report.findings.push(Finding::UnsupportedFormat { + format: manifest.format.clone(), + format_version: manifest.format_version, + }); + return report; + } + if canonical_json(&manifest).ok().as_deref() != Some(manifest_bytes.as_slice()) { + report.findings.push(Finding::ManifestNotCanonical { + path: ARCHIVE_MANIFEST_PATH.to_owned(), + }); + } + report.installation_id = Some(manifest.installation_id.clone()); + report.exported_at = Some(manifest.exported_at); + report.scope = Some(manifest.scope); + + let ordered = manifest.units.windows(2).all(|pair| { + (pair[0].record_id, pair[0].version_number) < (pair[1].record_id, pair[1].version_number) + }); + if !ordered { + report.findings.push(Finding::UnitsOutOfOrder); + } + let mut listed: BTreeSet = BTreeSet::new(); + listed.insert(ARCHIVE_MANIFEST_PATH.to_owned()); + for entry in &manifest.units { + let expected = unit_path(entry.record_id, entry.version_number); + if entry.path != expected { + report.findings.push(Finding::UnitPathUnexpected { + path: entry.path.clone(), + expected, + }); + } + listed.insert(format!("{}/{RECORD_FILE}", entry.path)); + listed.insert(format!("{}/{UNIT_MANIFEST_FILE}", entry.path)); + } + for name in &names { + if !listed.contains(name) { + report + .findings + .push(Finding::UnlistedEntry { path: name.clone() }); + } + } + let by_identity: BTreeMap<(i64, i64), &UnitEntry> = manifest + .units + .iter() + .map(|entry| ((entry.record_id, entry.version_number), entry)) + .collect(); + for entry in &manifest.units { + report + .units + .push(verify_unit(&mut archive, &manifest, entry, &by_identity)); + } + report +} + +#[allow(clippy::too_many_lines)] +fn verify_unit( + archive: &mut Archive<'_>, + manifest: &ArchiveManifest, + entry: &UnitEntry, + by_identity: &BTreeMap<(i64, i64), &UnitEntry>, +) -> UnitReport { + let mut findings = Vec::new(); + let record_path = format!("{}/{RECORD_FILE}", entry.path); + let manifest_path = format!("{}/{UNIT_MANIFEST_FILE}", entry.path); + + match read_entry(archive, &manifest_path) { + Ok(Some(bytes)) => match serde_json::from_slice::(&bytes) { + Ok(unit) => { + if unit.format != UNIT_FORMAT || unit.format_version != FORMAT_VERSION { + findings.push(Finding::UnsupportedFormat { + format: unit.format.clone(), + format_version: unit.format_version, + }); + } + if canonical_json(&unit).ok().as_deref() != Some(bytes.as_slice()) { + findings.push(Finding::ManifestNotCanonical { + path: manifest_path.clone(), + }); + } + let disagreements: [(&'static str, bool); 8] = [ + ( + "installation_id", + unit.installation_id != manifest.installation_id, + ), + ("exported_at", unit.exported_at != manifest.exported_at), + ("record_id", unit.record_id != entry.record_id), + ( + "version_number", + unit.version_number != entry.version_number, + ), + ("record_schema", unit.record_schema != entry.record_schema), + ("content_hash", unit.content_hash != entry.content_hash), + ("chain_hash", unit.chain_hash != entry.chain_hash), + ( + "predecessor_content_hash", + unit.predecessor_content_hash != entry.predecessor_content_hash, + ), + ]; + for (member, disagrees) in disagreements { + if disagrees { + findings.push(Finding::UnitManifestDisagrees { member }); + } + } + } + Err(err) => findings.push(Finding::UnitManifestUnreadable { + detail: err.to_string(), + }), + }, + Ok(None) => findings.push(Finding::MissingEntry { + path: manifest_path.clone(), + }), + Err(detail) => findings.push(Finding::EntryUnreadable { + path: manifest_path.clone(), + detail, + }), + } + + match read_entry(archive, &record_path) { + Ok(Some(bytes)) => { + if canonical::content_hash_hex(&bytes) != entry.content_hash { + findings.push(Finding::ContentHashMismatch); + } + match serde_json::from_slice::(&bytes) { + Ok(envelope) => { + match canonical::canonical_bytes(&envelope) { + Ok(again) if again == bytes => {} + Ok(_) => findings.push(Finding::NotCanonical { + detail: "re-serialization differs from the stored bytes".to_owned(), + }), + Err(err) => findings.push(Finding::NotCanonical { + detail: err.to_string(), + }), + } + let predecessor = entry + .predecessor_content_hash + .as_ref() + .map_or(Value::Null, |hash| Value::String(hash.clone())); + let disagreements: [(&'static str, bool); 6] = [ + ("record.id", envelope["record"]["id"] != entry.record_id), + ( + "record.version_number", + envelope["record"]["version_number"] != entry.version_number, + ), + ( + "record.record_schema", + envelope["record"]["record_schema"] != entry.record_schema, + ), + ( + "record.predecessor_content_hash", + envelope["record"]["predecessor_content_hash"] != predecessor, + ), + ( + "instance", + envelope["instance"] != manifest.installation_id.as_str(), + ), + ( + "canonicalization", + envelope["canonicalization"] != canonical::CANONICALIZATION, + ), + ]; + for (member, disagrees) in disagreements { + if disagrees { + findings.push(Finding::EnvelopeDisagrees { member }); + } + } + } + Err(err) => findings.push(Finding::NotCanonical { + detail: format!("not JSON: {err}"), + }), + } + match canonical::chain_hash_hex(entry.predecessor_content_hash.as_deref(), &bytes) { + Ok(chain) if chain == entry.chain_hash => {} + _ => findings.push(Finding::ChainHashMismatch), + } + } + Ok(None) => findings.push(Finding::MissingEntry { + path: record_path.clone(), + }), + Err(detail) => findings.push(Finding::EntryUnreadable { + path: record_path.clone(), + detail, + }), + } + + if (entry.version_number == 1) != entry.predecessor_content_hash.is_none() { + findings.push(Finding::LineageShape); + } + let predecessor = match &entry.predecessor_content_hash { + None => PredecessorLink::None, + Some(hash) => match entry + .version_number + .checked_sub(1) + .and_then(|number| by_identity.get(&(entry.record_id, number))) + { + Some(previous) => { + if previous.content_hash != *hash { + findings.push(Finding::PredecessorMismatch); + } + PredecessorLink::Linked + } + None => PredecessorLink::NotInExport, + }, + }; + + UnitReport { + path: entry.path.clone(), + record_id: entry.record_id, + version_number: entry.version_number, + record_schema: entry.record_schema, + predecessor, + findings, + } +} + +/// Reads one entry: `Ok(None)` when absent, `Err(detail)` when the +/// container cannot deliver it (a CRC mismatch included). +fn read_entry( + archive: &mut Archive<'_>, + name: &str, +) -> std::result::Result>, String> { + match archive.by_name(name) { + Ok(mut file) => { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes) + .map_err(|err| err.to_string())?; + Ok(Some(bytes)) + } + Err(zip::result::ZipError::FileNotFound) => Ok(None), + Err(err) => Err(err.to_string()), + } +} diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs new file mode 100644 index 0000000..a54599d --- /dev/null +++ b/crates/consolebook-server/tests/record_export.rs @@ -0,0 +1,1163 @@ +//! Milestone 5 slice 1: structured record exports — the stored canonical +//! bytes travel verbatim beside manifests, archives are deterministic, +//! verification needs only the archive and names every finding, scopes +//! follow the read rules that already exist, the API delivers the +//! documented bytes, and the CLI verifier works from the file alone. +//! Every fixture is invented. + +use std::io::{Cursor, Read, Write}; + +use axum::body::Body; +use axum::http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, COOKIE, SET_COOKIE}; +use axum::http::{HeaderMap, Request, StatusCode}; +use consolebook_server::capabilities::RoleBundle; +use consolebook_server::draft_content::{self, DraftContent, NarrativeEntry, RatingEntry}; +use consolebook_server::programs::{ + self, AnchorDef, CompetencyDef, FormCompetencyDef, FormDef, NarrativeDef, PolicyDef, + RecordType, ScaleDef, ScaleKind, VersionContent, +}; +use consolebook_server::record_export::{ + self, ARCHIVE_FORMAT, ARCHIVE_MANIFEST_PATH, ArchiveManifest, ExportRefusal, FORMAT_VERSION, + Finding, PredecessorLink, Scope, UNIT_FORMAT, UnitManifest, +}; +use consolebook_server::training_sessions::{self, Disposition, SessionInput}; +use consolebook_server::{ + amendments, assignments, canonical, data_dir::DataDir, enrollments, evaluation_drafts, + finalization, setup, storage, users, +}; +use http_body_util::BodyExt; +use tower::ServiceExt; +use zip::write::{SimpleFileOptions, ZipWriter}; + +const PASSWORD: &str = "invented-passphrase-1"; + +/// 2026-09-01T19:00:00Z, the instant every deterministic export below +/// is stamped with. +const EXPORTED_AT: i64 = 1_788_289_200; + +const OPEN_POLICY: PolicyDef = PolicyDef { + review_approved: false, + required_narratives: false, + ratings_complete: false, +}; + +struct Fixture { + _tmp: tempfile::TempDir, + pool: sqlx::SqlitePool, + admin_id: i64, +} + +impl Fixture { + async fn new() -> Self { + let tmp = tempfile::tempdir().expect("create temp dir"); + let data_dir = DataDir::new(tmp.path().join("data")); + data_dir.ensure_layout().expect("create layout"); + let pool = storage::open(&data_dir.database()).await.expect("open"); + let code = setup::issue_setup_code(&pool) + .await + .expect("issue") + .expect("uninitialized") + .0; + let admin_id = setup::initialize( + &pool, + &code.raw, + "Example County Communications", + "avery.admin", + "Avery Admin", + PASSWORD, + ) + .await + .expect("initialize") + .expect("accepted"); + Self { + _tmp: tmp, + pool, + admin_id, + } + } + + fn app(&self) -> axum::Router { + consolebook_server::http::router(consolebook_server::http::AppState { + pool: self.pool.clone(), + }) + } + + async fn login(&self, username: &str) -> String { + let response = self + .app() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/auth/login") + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::json!({ "username": username, "password": PASSWORD }) + .to_string(), + )) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(response.status(), StatusCode::OK, "login {username}"); + let cookie = response + .headers() + .get(SET_COOKIE) + .expect("cookie") + .to_str() + .expect("ascii"); + let (pair, _) = cookie.split_once(';').expect("attrs"); + pair.split_once('=').expect("pair").1.to_string() + } + + async fn user_with_role(&self, username: &str, display_name: &str, role: RoleBundle) -> i64 { + let created = users::create_with_reset_code( + &self.pool, + self.admin_id, + username, + display_name, + "", + "", + role, + ) + .await + .expect("create") + .expect("accepted"); + assert_eq!( + users::use_reset_code(&self.pool, username, &created.reset_code.raw, PASSWORD) + .await + .expect("reset"), + users::ResetOutcome::Done + ); + created.id + } + + /// The stored row of one version: bytes and both hashes. + async fn version_row(&self, record_id: i64, number: i64) -> (Vec, String, String) { + sqlx::query_as( + "SELECT canonical_bytes, content_hash, chain_hash FROM evaluation_version + WHERE evaluation_record_id = ?1 AND version_number = ?2", + ) + .bind(record_id) + .bind(number) + .fetch_one(&self.pool) + .await + .expect("version row") + } + + async fn audit_count(&self, subject_kind: Option<&str>) -> i64 { + sqlx::query_scalar( + "SELECT COUNT(*) FROM audit_event + WHERE kind = 'record_exported' AND subject_kind IS ?1", + ) + .bind(subject_kind) + .fetch_one(&self.pool) + .await + .expect("count") + } +} + +/// A GET whose body is delivered raw, for downloads. +async fn raw_get(app: axum::Router, uri: &str, cookie: &str) -> (StatusCode, HeaderMap, Vec) { + let response = app + .oneshot( + Request::builder() + .uri(uri) + .header( + COOKIE, + format!("{}={}", consolebook_server::http::SESSION_COOKIE, cookie), + ) + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + let status = response.status(); + let headers = response.headers().clone(); + let bytes = response + .into_body() + .collect() + .await + .expect("body") + .to_bytes() + .to_vec(); + (status, headers, bytes) +} + +/// Invented single-form program with every completion rule off, so +/// records seal from the working copy as soon as they are authored. +fn program(name: &str) -> VersionContent { + VersionContent { + name: name.to_owned(), + label: "2026 rev A".to_owned(), + description: "Invented program for export tests.".to_owned(), + phases: Vec::new(), + phase_transitions: Vec::new(), + competencies: vec![CompetencyDef { + category: "Call processing".to_owned(), + name: "Emergency Call Interrogation".to_owned(), + description: "Obtains and verifies location, callback, and nature.".to_owned(), + tasks: Vec::new(), + citations: Vec::new(), + }], + rating_scales: vec![ScaleDef { + name: "Standard 1-7".to_owned(), + kind: ScaleKind::AnchoredNumeric, + min_value: Some(1), + max_value: Some(7), + anchors: vec![AnchorDef { + value: 4, + label: "Meets standards".to_owned(), + definition: "To the invented standard.".to_owned(), + }], + }], + rating_modifiers: Vec::new(), + evaluation_forms: vec![FormDef { + record_type: RecordType::DailyReport, + name: "Daily Observation Report".to_owned(), + instructions: "Rate observed performance.".to_owned(), + competencies: vec![FormCompetencyDef { + competency: "Emergency Call Interrogation".to_owned(), + rating_scale: "Standard 1-7".to_owned(), + }], + narratives: vec![NarrativeDef { + prompt: "Most acceptable performance.".to_owned(), + required: false, + }], + }], + citations: Vec::new(), + finalization_policy: OPEN_POLICY, + } +} + +#[allow(clippy::struct_field_names)] +struct Seeded { + version_id: i64, + enrollment_id: i64, + record_id: i64, + taylor_id: i64, + jordan_id: i64, + casey_id: i64, +} + +/// A published program, an enrolled trainee, an assigned trainer, a +/// coordinator, and one daily record sealed twice: version 1, then an +/// amendment sealed as version 2 chained to it. +async fn seed(fx: &Fixture, suffix: &str) -> Seeded { + let content = program(&format!("Example County Program {suffix}")); + let program_id = programs::create_program(&fx.pool, fx.admin_id, &content.name) + .await + .expect("create program") + .expect("accepted"); + let version_id = programs::create_version(&fx.pool, fx.admin_id, program_id, &content) + .await + .expect("create version") + .expect("accepted"); + programs::publish_version(&fx.pool, fx.admin_id, version_id) + .await + .expect("publish") + .expect("accepted"); + let taylor_id = fx + .user_with_role( + &format!("taylor.{suffix}"), + "Taylor Trainee", + RoleBundle::Trainee, + ) + .await; + let jordan_id = fx + .user_with_role( + &format!("jordan.{suffix}"), + "Jordan Trainer", + RoleBundle::Trainer, + ) + .await; + let casey_id = fx + .user_with_role( + &format!("casey.{suffix}"), + "Casey Coordinator", + RoleBundle::Coordinator, + ) + .await; + let enrollment_id = enrollments::enroll(&fx.pool, fx.admin_id, version_id, taylor_id) + .await + .expect("call") + .expect("enrolled"); + assignments::create(&fx.pool, fx.admin_id, enrollment_id, jordan_id) + .await + .expect("call") + .expect("assigned"); + let record_id = draft_for(fx, jordan_id, enrollment_id, "2026-06-02").await; + let s = Seeded { + version_id, + enrollment_id, + record_id, + taylor_id, + jordan_id, + casey_id, + }; + seal_twice(fx, &s).await; + s +} + +async fn draft_for(fx: &Fixture, trainer_id: i64, enrollment_id: i64, date: &str) -> i64 { + let session_id = training_sessions::create( + &fx.pool, + trainer_id, + enrollment_id, + &SessionInput { + business_date: date.to_owned(), + timezone: "America/Chicago".to_owned(), + local_start: format!("{date}T07:00"), + local_end: Some(format!("{date}T15:00")), + disposition: Some(Disposition::Completed), + phase_id: None, + trainer_user_ids: Vec::new(), + }, + ) + .await + .expect("call") + .expect("created"); + evaluation_drafts::create(&fx.pool, trainer_id, session_id, None) + .await + .expect("call") + .expect("created") +} + +async fn author_and_seal(fx: &Fixture, s: &Seeded, value: i64, text: &str) { + let workspace = evaluation_drafts::workspace(&fx.pool, s.jordan_id, s.record_id) + .await + .expect("call") + .expect("readable"); + let revision = draft_content::save( + &fx.pool, + s.jordan_id, + s.record_id, + workspace.detail.revision, + &DraftContent { + ratings: vec![RatingEntry { + form_competency_id: workspace.form.competencies[0].form_competency_id, + value: Some(value), + not_observed: false, + modifier_ids: Vec::new(), + }], + narratives: vec![NarrativeEntry { + form_narrative_id: workspace.form.narratives[0].form_narrative_id, + text: text.to_owned(), + }], + }, + ) + .await + .expect("call") + .expect("saved"); + finalization::finalize(&fx.pool, s.casey_id, s.record_id, revision) + .await + .expect("call") + .expect("sealed"); +} + +async fn seal_twice(fx: &Fixture, s: &Seeded) { + author_and_seal(fx, s, 3, "The invented initial entry.").await; + amendments::open( + &fx.pool, + s.casey_id, + s.record_id, + "The invented rating was entered one point low.", + ) + .await + .expect("call") + .expect("opened"); + author_and_seal(fx, s, 4, "Corrected the invented rating with context.").await; +} + +async fn export(fx: &Fixture, actor: i64, scope: Scope) -> Vec { + record_export::export_at(&fx.pool, actor, scope, EXPORTED_AT) + .await + .expect("call") + .expect("exported") + .bytes +} + +async fn refusal(fx: &Fixture, actor: i64, scope: Scope) -> ExportRefusal { + record_export::export_at(&fx.pool, actor, scope, EXPORTED_AT) + .await + .expect("call") + .expect_err("refused") +} + +/// Every entry of an archive in container order. +fn entries(bytes: &[u8]) -> Vec<(String, Vec)> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).expect("zip"); + let mut out = Vec::new(); + for index in 0..archive.len() { + let mut file = archive.by_index(index).expect("entry"); + let mut content = Vec::new(); + file.read_to_end(&mut content).expect("read"); + out.push((file.name().to_owned(), content)); + } + out +} + +/// A well-formed container holding exactly `entries`: the way a +/// tamperer who understands ZIP would repack an export. +fn repack(entries: &[(String, Vec)]) -> Vec { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for (name, content) in entries { + writer.start_file(name.as_str(), options).expect("start"); + writer.write_all(content).expect("write"); + } + writer.finish().expect("finish").into_inner() +} + +fn edit_json(bytes: &[u8], edit: impl FnOnce(&mut serde_json::Value)) -> Vec { + let mut value: serde_json::Value = serde_json::from_slice(bytes).expect("json"); + edit(&mut value); + canonical::canonical_bytes(&value).expect("canonical") +} + +fn entry<'a>(entries: &'a [(String, Vec)], name: &str) -> &'a [u8] { + &entries + .iter() + .find(|(entry_name, _)| entry_name == name) + .unwrap_or_else(|| panic!("entry {name}")) + .1 +} + +fn with_entry( + entries: &[(String, Vec)], + name: &str, + replace: impl FnOnce(&[u8]) -> Option>, +) -> Vec<(String, Vec)> { + let mut replace = Some(replace); + entries + .iter() + .filter_map(|(entry_name, content)| { + if entry_name == name { + replace.take().expect("one edit")(content) + } else { + Some(content.clone()) + } + .map(|content| (entry_name.clone(), content)) + }) + .collect() +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn exports_carry_stored_bytes_verbatim_and_verify() { + let fx = Fixture::new().await; + let s = seed(&fx, "verbatim").await; + let installation_id = storage::installation_id(&fx.pool).await.expect("id"); + let (v1_bytes, v1_content, v1_chain) = fx.version_row(s.record_id, 1).await; + let (v2_bytes, v2_content, v2_chain) = fx.version_row(s.record_id, 2).await; + + // One version: the archive lays out exactly the documented entries, + // the record bytes are the stored bytes, and both manifests carry + // the stored identity and fingerprints. + let scope = Scope::Version { + record_id: s.record_id, + version_number: 2, + }; + let exported = record_export::export_at(&fx.pool, s.casey_id, scope, EXPORTED_AT) + .await + .expect("call") + .expect("exported"); + assert_eq!( + exported.file_name, + format!("consolebook-record-{}-v2-20260901T190000Z.zip", s.record_id) + ); + assert_eq!(exported.unit_count, 1); + let unit_dir = format!("records/{}/v2", s.record_id); + let listed = entries(&exported.bytes); + let names: Vec<&str> = listed.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!( + names, + [ + ARCHIVE_MANIFEST_PATH.to_owned(), + format!("{unit_dir}/record.json"), + format!("{unit_dir}/manifest.json"), + ] + ); + assert_eq!( + entry(&listed, &format!("{unit_dir}/record.json")), + v2_bytes.as_slice(), + "the record bytes are the stored bytes, not a re-serialization" + ); + let manifest_bytes = entry(&listed, ARCHIVE_MANIFEST_PATH); + let manifest: ArchiveManifest = serde_json::from_slice(manifest_bytes).expect("manifest"); + assert_eq!(manifest.format, ARCHIVE_FORMAT); + assert_eq!(manifest.format_version, FORMAT_VERSION); + assert_eq!(manifest.installation_id, installation_id); + assert_eq!(manifest.exported_at, EXPORTED_AT); + assert_eq!(manifest.scope, scope); + assert_eq!(manifest.units.len(), 1); + let unit = &manifest.units[0]; + assert_eq!(unit.path, unit_dir); + assert_eq!(unit.record_id, s.record_id); + assert_eq!(unit.version_number, 2); + assert_eq!(unit.record_schema, canonical::RECORD_SCHEMA); + assert_eq!(unit.content_hash, v2_content); + assert_eq!(unit.chain_hash, v2_chain); + assert_eq!( + unit.predecessor_content_hash.as_deref(), + Some(v1_content.as_str()) + ); + // Manifests are canonical JSON: members sorted, compact. + let manifest_value: serde_json::Value = serde_json::from_slice(manifest_bytes).expect("json"); + assert_eq!( + canonical::canonical_bytes(&manifest_value).expect("canonical"), + manifest_bytes + ); + let unit_manifest: UnitManifest = + serde_json::from_slice(entry(&listed, &format!("{unit_dir}/manifest.json"))) + .expect("unit manifest"); + assert_eq!(unit_manifest.format, UNIT_FORMAT); + assert_eq!(unit_manifest.format_version, FORMAT_VERSION); + assert_eq!(unit_manifest.installation_id, installation_id); + assert_eq!(unit_manifest.exported_at, EXPORTED_AT); + assert_eq!(unit_manifest.record_id, s.record_id); + assert_eq!(unit_manifest.version_number, 2); + assert_eq!(unit_manifest.content_hash, v2_content); + assert_eq!(unit_manifest.chain_hash, v2_chain); + assert_eq!( + unit_manifest.predecessor_content_hash.as_deref(), + Some(v1_content.as_str()) + ); + + // Verification from the archive alone: the chain hash recomputes + // from the carried predecessor hash, and the predecessor itself is + // honestly reported as not in this export. + let report = record_export::verify_archive(&exported.bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!( + report.installation_id.as_deref(), + Some(installation_id.as_str()) + ); + assert_eq!(report.exported_at, Some(EXPORTED_AT)); + assert_eq!(report.scope, Some(scope)); + assert_eq!(report.units.len(), 1); + assert_eq!(report.units[0].predecessor, PredecessorLink::NotInExport); + + // Deterministic: the same scope at the same instant is byte-identical. + let again = export(&fx, s.casey_id, scope).await; + assert_eq!(again, exported.bytes); + let later = export(&fx, s.casey_id, scope).await; + assert_eq!(later, exported.bytes); + + // The whole record: both versions in order, the successor linked to + // its predecessor within the archive. + let record_scope = Scope::Record { + record_id: s.record_id, + }; + let bytes = export(&fx, s.casey_id, record_scope).await; + let listed = entries(&bytes); + assert_eq!(listed.len(), 5); + assert_eq!( + entry(&listed, &format!("records/{}/v1/record.json", s.record_id)), + v1_bytes.as_slice() + ); + assert_eq!( + entry(&listed, &format!("records/{}/v2/record.json", s.record_id)), + v2_bytes.as_slice() + ); + let manifest: ArchiveManifest = + serde_json::from_slice(entry(&listed, ARCHIVE_MANIFEST_PATH)).expect("manifest"); + assert_eq!(manifest.scope, record_scope); + assert_eq!(manifest.units.len(), 2); + assert_eq!(manifest.units[0].version_number, 1); + assert_eq!(manifest.units[0].chain_hash, v1_chain); + assert_eq!(manifest.units[0].predecessor_content_hash, None); + assert_eq!(manifest.units[1].version_number, 2); + let report = record_export::verify_archive(&bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.units[0].predecessor, PredecessorLink::None); + assert_eq!(report.units[1].predecessor, PredecessorLink::Linked); + + // Every export is audited with its subject and never with content. + assert_eq!(fx.audit_count(Some("record")).await, 4); + let bytes = export( + &fx, + s.jordan_id, + Scope::Enrollment { + enrollment_id: s.enrollment_id, + }, + ) + .await; + assert!(record_export::verify_archive(&bytes).verified()); + assert_eq!(fx.audit_count(Some("enrollment")).await, 1); + let bytes = export(&fx, fx.admin_id, Scope::Installation).await; + let report = record_export::verify_archive(&bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.scope, Some(Scope::Installation)); + assert_eq!(report.units.len(), 2); + assert_eq!(fx.audit_count(None).await, 1); + let with_content: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM audit_event WHERE kind = 'record_exported' + AND (subject_user_id IS NULL) != (subject_kind IS NULL)", + ) + .fetch_one(&fx.pool) + .await + .expect("count"); + assert_eq!(with_content, 0, "subject and trainee travel together"); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn verification_names_every_finding() { + let fx = Fixture::new().await; + let s = seed(&fx, "findings").await; + let original = export( + &fx, + s.casey_id, + Scope::Record { + record_id: s.record_id, + }, + ) + .await; + let listed = entries(&original); + let v1 = format!("records/{}/v1", s.record_id); + let v2 = format!("records/{}/v2", s.record_id); + let (v1_bytes, _, _) = fx.version_row(s.record_id, 1).await; + + // Not an archive at all. + let report = record_export::verify_archive(b"not a zip archive"); + assert!(!report.verified()); + assert!(matches!(report.findings[0], Finding::NotAnArchive { .. })); + + // Repacked with altered record content: the content hash no longer + // covers the bytes, and neither does the chain hash. + let altered = repack(&with_entry( + &listed, + &format!("{v2}/record.json"), + |bytes| { + let text = String::from_utf8(bytes.to_vec()).expect("utf-8"); + Some( + text.replace("Corrected the invented", "Corrected the inveNted") + .into_bytes(), + ) + }, + )); + let report = record_export::verify_archive(&altered); + assert!(!report.verified()); + assert!(report.findings.is_empty(), "{report:?}"); + assert!(report.units[0].verified()); + let findings = &report.units[1].findings; + assert!( + findings.contains(&Finding::ContentHashMismatch), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::ChainHashMismatch), + "{findings:?}" + ); + assert!( + !findings + .iter() + .any(|f| matches!(f, Finding::NotCanonical { .. })), + "the altered text is still canonical JSON: {findings:?}" + ); + + // A byte flipped inside the container without repacking fails too: + // the container's own checksum or the content hash catches it. + let mut flipped = original.clone(); + let needle = b"Corrected the invented"; + let at = flipped + .windows(needle.len()) + .position(|window| window == needle) + .expect("record text in the archive"); + flipped[at] ^= 0x20; + assert!(!record_export::verify_archive(&flipped).verified()); + + // The archive manifest claims a different content hash for version + // 1: the bytes disagree, the unit manifest disagrees, and version + // 2's predecessor no longer matches the version 1 in the archive. + let other_hash = "0".repeat(64); + let swapped = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"][0]["content_hash"] = serde_json::Value::String(other_hash.clone()); + })) + })); + let report = record_export::verify_archive(&swapped); + assert!(!report.verified()); + assert!(report.findings.is_empty(), "{report:?}"); + let findings = &report.units[0].findings; + assert!( + findings.contains(&Finding::ContentHashMismatch), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::UnitManifestDisagrees { + member: "content_hash" + }), + "{findings:?}" + ); + assert!( + !findings.contains(&Finding::ChainHashMismatch), + "the chain hash still covers the bytes: {findings:?}" + ); + let findings = &report.units[1].findings; + assert!( + findings.contains(&Finding::PredecessorMismatch), + "{findings:?}" + ); + assert_eq!(report.units[1].predecessor, PredecessorLink::Linked); + + // A unit's record bytes replaced by another version's: every + // identity member the envelope itself carries disagrees. + let replaced = repack(&with_entry(&listed, &format!("{v2}/record.json"), |_| { + Some(v1_bytes.clone()) + })); + let report = record_export::verify_archive(&replaced); + let findings = &report.units[1].findings; + assert!( + findings.contains(&Finding::ContentHashMismatch), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::EnvelopeDisagrees { + member: "record.version_number" + }), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::EnvelopeDisagrees { + member: "record.predecessor_content_hash" + }), + "{findings:?}" + ); + + // Missing and unlisted entries. + let missing = repack(&with_entry(&listed, &format!("{v1}/record.json"), |_| None)); + let report = record_export::verify_archive(&missing); + assert!( + report.units[0].findings.contains(&Finding::MissingEntry { + path: format!("{v1}/record.json") + }), + "{report:?}" + ); + let mut extra = listed.clone(); + extra.push(("README.txt".to_owned(), b"nothing to see here".to_vec())); + let report = record_export::verify_archive(&repack(&extra)); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::UnlistedEntry { + path: "README.txt".to_owned() + }), + "{report:?}" + ); + assert!(report.units.iter().all(record_export::UnitReport::verified)); + + // A manifest that parses but is not canonical (pretty-printed) is a + // finding: the format fixes the bytes, not only the members. + let pretty = repack(&with_entry( + &listed, + &format!("{v2}/manifest.json"), + |bytes| { + let value: serde_json::Value = serde_json::from_slice(bytes).expect("json"); + Some(serde_json::to_vec_pretty(&value).expect("pretty")) + }, + )); + let report = record_export::verify_archive(&pretty); + assert!( + report.units[1] + .findings + .contains(&Finding::ManifestNotCanonical { + path: format!("{v2}/manifest.json") + }), + "{report:?}" + ); + + // An unknown format version is refused by name, not guessed at. + let future = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["format_version"] = serde_json::Value::from(2); + })) + })); + let report = record_export::verify_archive(&future); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::UnsupportedFormat { + format: ARCHIVE_FORMAT.to_owned(), + format_version: 2 + }), + "{report:?}" + ); + assert!(report.units.is_empty()); + + // Units out of order, with paths that do not derive from identity. + let reordered = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + let units = manifest["units"].as_array_mut().expect("units"); + units.swap(0, 1); + })) + })); + let report = record_export::verify_archive(&reordered); + assert!( + report.findings.contains(&Finding::UnitsOutOfOrder), + "{report:?}" + ); + let renamed = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"][1]["path"] = serde_json::Value::String("records/elsewhere".into()); + })) + })); + let report = record_export::verify_archive(&renamed); + assert!( + report.findings.contains(&Finding::UnitPathUnexpected { + path: "records/elsewhere".to_owned(), + expected: v2.clone() + }), + "{report:?}" + ); + + // Lineage shape: a version 2 claiming no predecessor. + let orphan = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"][1]["predecessor_content_hash"] = serde_json::Value::Null; + })) + })); + let report = record_export::verify_archive(&orphan); + let findings = &report.units[1].findings; + assert!(findings.contains(&Finding::LineageShape), "{findings:?}"); + assert!( + findings.contains(&Finding::ChainHashMismatch), + "{findings:?}" + ); + assert_eq!(report.units[1].predecessor, PredecessorLink::None); + + // The untouched original still verifies after all that. + assert!(record_export::verify_archive(&original).verified()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn scopes_follow_the_read_rules() { + let fx = Fixture::new().await; + let s = seed(&fx, "scopes").await; + let outsider = fx + .user_with_role("robin.scopes", "Robin Outsider", RoleBundle::Trainer) + .await; + let version = Scope::Version { + record_id: s.record_id, + version_number: 1, + }; + let record = Scope::Record { + record_id: s.record_id, + }; + let enrollment = Scope::Enrollment { + enrollment_id: s.enrollment_id, + }; + + // A version or record exports for whoever may read the record: the + // assigned trainer, the coordinator, and the trainee on their own + // finalized record — every retained version included. A trainer + // outside the scope is refused. + for actor in [s.jordan_id, s.casey_id, s.taylor_id] { + let bytes = export(&fx, actor, record).await; + let report = record_export::verify_archive(&bytes); + assert!(report.verified()); + assert_eq!(report.units.len(), 2, "actor {actor} sees both versions"); + export(&fx, actor, version).await; + } + assert_eq!( + refusal(&fx, outsider, version).await, + ExportRefusal::CapabilityRequired + ); + assert_eq!( + refusal(&fx, outsider, record).await, + ExportRefusal::CapabilityRequired + ); + + // An enrollment exports for whoever may read its training history; + // the trainee's own-record grant is not that. + export(&fx, s.jordan_id, enrollment).await; + export(&fx, s.casey_id, enrollment).await; + export(&fx, fx.admin_id, enrollment).await; + assert_eq!( + refusal(&fx, outsider, enrollment).await, + ExportRefusal::CapabilityRequired + ); + assert_eq!( + refusal(&fx, s.taylor_id, enrollment).await, + ExportRefusal::CapabilityRequired + ); + + // The whole installation takes export_records: the administrator + // bundle carries it, the coordinator bundle does not. + export(&fx, fx.admin_id, Scope::Installation).await; + assert_eq!( + refusal(&fx, s.casey_id, Scope::Installation).await, + ExportRefusal::CapabilityRequired + ); + let summary = record_export::summary(&fx.pool, fx.admin_id) + .await + .expect("call") + .expect("summarized"); + assert_eq!((summary.record_count, summary.version_count), (1, 2)); + assert_eq!( + record_export::summary(&fx.pool, s.casey_id) + .await + .expect("call"), + Err(ExportRefusal::CapabilityRequired) + ); + + // Unknown identities and empty scopes are typed, never an empty + // archive presented as complete. + assert_eq!( + refusal(&fx, s.casey_id, Scope::Record { record_id: 9999 }).await, + ExportRefusal::NoSuchRecord + ); + assert_eq!( + refusal( + &fx, + s.casey_id, + Scope::Version { + record_id: s.record_id, + version_number: 7, + } + ) + .await, + ExportRefusal::NoSuchVersion + ); + assert_eq!( + refusal( + &fx, + s.casey_id, + Scope::Enrollment { + enrollment_id: 9999 + } + ) + .await, + ExportRefusal::NoSuchEnrollment + ); + let unfinalized = draft_for(&fx, s.jordan_id, s.enrollment_id, "2026-06-03").await; + assert_eq!( + refusal( + &fx, + s.jordan_id, + Scope::Record { + record_id: unfinalized + } + ) + .await, + ExportRefusal::NothingToExport + ); + let riley = fx + .user_with_role("riley.scopes", "Riley Trainee", RoleBundle::Trainee) + .await; + let empty_enrollment = enrollments::enroll(&fx.pool, fx.admin_id, s.version_id, riley) + .await + .expect("call") + .expect("enrolled"); + assert_eq!( + refusal( + &fx, + fx.admin_id, + Scope::Enrollment { + enrollment_id: empty_enrollment + } + ) + .await, + ExportRefusal::NothingToExport + ); + // Refusals leave no export audit behind. + let audited: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM audit_event WHERE kind = 'record_exported'") + .fetch_one(&fx.pool) + .await + .expect("count"); + assert_eq!(audited, 10); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn export_api_delivers_the_documented_bytes() { + let fx = Fixture::new().await; + let s = seed(&fx, "api").await; + let casey = fx.login("casey.api").await; + let admin = fx.login("avery.admin").await; + + let (status, headers, bytes) = raw_get( + fx.app(), + &format!("/api/drafts/{}/versions/2/export", s.record_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers + .get(CONTENT_TYPE) + .expect("type") + .to_str() + .expect("ascii"), + "application/zip" + ); + let disposition = headers + .get(CONTENT_DISPOSITION) + .expect("disposition") + .to_str() + .expect("ascii"); + assert!( + disposition.starts_with(&format!( + "attachment; filename=\"consolebook-record-{}-v2-", + s.record_id + )) && disposition.ends_with("Z.zip\""), + "got: {disposition}" + ); + let report = record_export::verify_archive(&bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.units.len(), 1); + + let (status, _, bytes) = raw_get( + fx.app(), + &format!("/api/drafts/{}/export", s.record_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(record_export::verify_archive(&bytes).units.len(), 2); + let (status, _, bytes) = raw_get( + fx.app(), + &format!("/api/enrollments/{}/export", s.enrollment_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(record_export::verify_archive(&bytes).verified()); + + // The installation scope answers to export_records only. + let (status, _, body) = raw_get(fx.app(), "/api/exports/records", &casey).await; + assert_eq!(status, StatusCode::FORBIDDEN); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "capability_required"); + let (status, _, bytes) = raw_get(fx.app(), "/api/exports/records", &admin).await; + assert_eq!(status, StatusCode::OK); + let report = record_export::verify_archive(&bytes); + assert!(report.verified()); + assert_eq!(report.scope, Some(Scope::Installation)); + let (status, _, body) = raw_get(fx.app(), "/api/exports/summary", &admin).await; + assert_eq!(status, StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["record_count"], 1); + assert_eq!(body["version_count"], 2); + let (status, _, _) = raw_get(fx.app(), "/api/exports/summary", &casey).await; + assert_eq!(status, StatusCode::FORBIDDEN); + + // Typed refusals over the wire. + let (status, _, body) = raw_get(fx.app(), "/api/enrollments/9999/export", &casey).await; + assert_eq!(status, StatusCode::NOT_FOUND); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "no_such_enrollment"); + let (status, _, body) = raw_get( + fx.app(), + &format!("/api/drafts/{}/versions/9/export", s.record_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "no_such_version"); + let unfinalized = draft_for(&fx, s.jordan_id, s.enrollment_id, "2026-06-03").await; + let (status, _, body) = raw_get( + fx.app(), + &format!("/api/drafts/{unfinalized}/export"), + &casey, + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "nothing_to_export"); + let outsider_cookie = { + fx.user_with_role("robin.api", "Robin Outsider", RoleBundle::Trainer) + .await; + fx.login("robin.api").await + }; + let (status, _, _) = raw_get( + fx.app(), + &format!("/api/drafts/{}/export", s.record_id), + &outsider_cookie, + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + let (status, _, _) = raw_get( + fx.app(), + &format!("/api/drafts/{}/export", s.record_id), + "not-a-session", + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn cli_verifies_from_the_file_alone() { + let fx = Fixture::new().await; + let s = seed(&fx, "cli").await; + let bytes = export( + &fx, + s.casey_id, + Scope::Record { + record_id: s.record_id, + }, + ) + .await; + let scratch = tempfile::tempdir().expect("scratch"); + let archive = scratch.path().join("export.zip"); + std::fs::write(&archive, &bytes).expect("write"); + // The data directory named here must never be touched: the archive + // carries everything the checks need. + let untouched = scratch.path().join("never-created"); + + let output = std::process::Command::new(env!("CARGO_BIN_EXE_consolebook-server")) + .args(["--data-dir"]) + .arg(&untouched) + .args(["export", "verify"]) + .arg(&archive) + .output() + .expect("run verifier"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(output.status.success(), "stdout: {stdout}"); + assert!(stdout.contains("verified 2 of 2 units"), "stdout: {stdout}"); + assert!(stdout.contains("predecessor linked"), "stdout: {stdout}"); + assert!(!untouched.exists(), "the verifier opened no data directory"); + + // A tampered file fails with a named finding and a failing exit code. + let listed = entries(&bytes); + let tampered = repack(&with_entry( + &listed, + &format!("records/{}/v1/record.json", s.record_id), + |content| { + let text = String::from_utf8(content.to_vec()).expect("utf-8"); + Some( + text.replace("initial entry", "initial entry, revised") + .into_bytes(), + ) + }, + )); + std::fs::write(&archive, &tampered).expect("write"); + let output = std::process::Command::new(env!("CARGO_BIN_EXE_consolebook-server")) + .args(["export", "verify"]) + .arg(&archive) + .output() + .expect("run verifier"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "stdout: {stdout}"); + assert!(stdout.contains("NOT VERIFIED"), "stdout: {stdout}"); + assert!( + stdout.contains("the content hash does not match the record bytes"), + "stdout: {stdout}" + ); + // Version 2 still verifies on its own: its chain covers the hash the + // manifest states for version 1, and that statement is unchanged; + // the altered bytes fail where they sit. + assert!( + stdout.contains("NOT VERIFIED: 1 of 2 units consistent, 0 archive finding(s)"), + "stdout: {stdout}" + ); + assert!( + stdout.contains(&format!("FAIL records/{}/v1", s.record_id)), + "stdout: {stdout}" + ); + assert!( + stdout.contains(&format!("ok records/{}/v2", s.record_id)), + "stdout: {stdout}" + ); +} diff --git a/docs/decisions/0014-record-export-format.md b/docs/decisions/0014-record-export-format.md new file mode 100644 index 0000000..b310532 --- /dev/null +++ b/docs/decisions/0014-record-export-format.md @@ -0,0 +1,140 @@ +# ADR 0014: Record export format and verification + +- **Status:** Accepted +- **Date:** 2026-09-01 + +## Context + +Milestone 5 makes an installation leave-able: "a center can leave with +all of its data and can prove recovery from a clean installation" +(`docs/roadmap.md`). Milestone 4 finished the record substrate — +canonical bytes with content and chain hashes (ADR 0011), successor +versions (ADR 0012), record schema 2 (ADR 0013) — but a finalized +version is readable only through the installation that holds it. +`docs/records-integrity.md` requires export round-trip tests and +honest verification wording, and #44 decision 1 settled the owner +choice: the stored canonical bytes travel verbatim beside a manifest, +multi-record exports are an archive of the same units plus an archive +manifest, and verification needs nothing but the export. This ADR +fixes the format (`docs/formats/record-export.md`), its scopes, who +may export, and what verification claims (#45; Milestone 5 slice 1). + +## Decision + +### Units carry stored bytes, never a re-serialization + +- A unit is one finalized version's `canonical_bytes` copied from + storage, beside a unit manifest naming the installation, the record + and version identity, the stored `record_schema`, both stored hashes, + the predecessor's content hash, the export instant, and the format + version. Schema-1 and schema-2 bytes travel as stored; nothing + upgrades, rewraps, or reformats a record on the way out. +- Manifests are themselves canonical JSON under the record format's + JCS subset, so an export is deterministic: the same scope exported + at the same instant is byte-identical. + +### Archives are ZIP containers, one format for every scope + +- Every export, a single version included, is a ZIP archive with + stored entries, a fixed entry order, fixed entry metadata, and an + archive manifest listing every unit with its identity and hashes. One + container means one reader, one verifier, and one document for all + four scopes: version, record (every retained version), enrollment + (every finalized version of its records), and installation. +- The archive holds nothing this format does not name; extra entries + are verification findings, so a tampered or repacked archive cannot + hide content beside the records. +- A scope with no finalized version is a typed refusal + (`nothing_to_export`), never an empty archive presented as a + complete export. + +### Verification from the export alone + +- The verifier recomputes the content hash over the bytes, proves the + bytes are canonical by re-serialization, cross-checks the envelope's + own `record` and `instance` members against the manifests, + recomputes the chain hash from the carried predecessor hash, and + links predecessors present in the same archive. It reports per unit + and per archive with typed findings; the verdict is `verified` only + when every check passes. +- It ships as a library function and as + `consolebook-server export verify `, which opens no data + directory — the artifact is checked wherever it landed. +- Honest limits (ADR 0010, ADR 0011): a verified export is internally + consistent with its stated fingerprints. It does not prove which + installation produced it, nor completeness against what that + installation held; the scope member is the exporter's statement of + intent. Signatures (the future signed mode) attach beside the hashes + without changing a record byte. + +### Export is a read in a portable shape + +- Authorization reuses the read contracts that exist rather than + adding parallel ones: a version or record exports for whoever may + read the record (workflow readers, and the trainee on their own + finalized record — every retained version, per ADR 0012); an + enrollment exports for whoever may read its training history; the + whole installation exports for `export_records` holders, the + explicit administrative authority PRINCIPLES.md 10 demands for + breadth. A unit contains exactly what its reader can already read + through the API; the capability gates breadth, not format. +- Every export is audited (`record_exported`) with actor and subject + and never with content, in the append-only audit trail + (`docs/records-integrity.md`). + +### Nothing is retained on disk + +- Exports are produced on request and delivered as downloads; this + slice writes nothing under `data/exports/`. A future persisted export + (scheduled exports, large archives) must register each file so + lawful disposition can reach derived exports in scope + (`docs/records-integrity.md` step 4); until then there are none. + +## Consequences + +### Positive + +- a record leaves the installation exactly as it was sealed, and any + reader of the format document can check it without Consolebook; +- one container and one verifier cover single versions, records, + enrollments, and the whole installation, and the trainee packet + (slice 2) can be built from the same units; +- determinism makes exports comparable and testable byte for byte, + the property `docs/records-integrity.md` asks of every record + representation; and +- verification wording carries the same honesty as in-database + verification — consistency, not tamper-proofing. + +### Costs + +- a new dependency (`zip`, without default features: stored entries + only) enters the one executable; +- the format version is shared by both manifests, so any manifest + change bumps it for every export; +- an archive pins the predecessor by content hash only, so an + exported successor without its predecessor verifies its own chain + hash but cannot prove the predecessor's bytes — reported as *not in + export*, never inferred; and +- streaming exports means nothing on disk to dispose of, and nothing + to resume: a very large installation export is a single response. + +## Rejected alternatives + +- **JSON re-serialization of the envelope (a "pretty" export):** the + hash is over the specified bytes; any re-serialization is either a + no-op (pointless) or a change (a record that no longer verifies). +- **A single JSON document embedding the bytes as strings or + base64:** embedding re-encodes the bytes, and base64 hides the + record from a human reader; the decision was units *beside* a + manifest, unchanged. +- **Tar or a bespoke container:** ZIP opens on every operator + platform without tooling; a bespoke container would need its own + reader in every verifier. +- **A separate `export_records` gate on every scope:** a trainee may + already read every retained version of their own record and a + reviewer every record in their scope; forcing an administrator to + produce those exports adds a gatekeeper without adding protection, + which is not what PRINCIPLES.md 10's explicit-authority rule is for. +- **Exporter identity in the manifest:** provenance of a file is the + audit trail's job; putting an operator's identity in every artifact + that leaves the system is personal data the record does not need. diff --git a/docs/domain-model.md b/docs/domain-model.md index feab28f..682ee0d 100644 --- a/docs/domain-model.md +++ b/docs/domain-model.md @@ -138,6 +138,12 @@ Acknowledgment means receipt, not agreement. A successor version requires a new An Amendment links an original finalized version to its successor and records the reason, authority, author, and timestamps. The original remains readable and exportable while retained. +## Exports + +### RecordExport + +A RecordExport is an archive of finalized EvaluationVersions as stored: each version's canonical bytes, unchanged, beside a manifest carrying the installation identity, record and version identity, record schema, both hashes, the predecessor's content hash, and the export instant (`docs/formats/record-export.md`, ADR 0014). Scopes are one version, one record, one enrollment, or the installation. Verifying an export needs nothing but the export; the verdict reports consistency with the stated fingerprints, never tamper-proofing. Every export is audited without record content. + ## Retention and disposition ### RetentionPolicy diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md new file mode 100644 index 0000000..5be43ca --- /dev/null +++ b/docs/formats/record-export.md @@ -0,0 +1,243 @@ +# Record Export Format + +Portable, verifiable exports of finalized evaluation versions (ADR 0014; +`docs/records-integrity.md`; #44 decision 1). +`consolebook-server/src/record_export.rs` implements it; +`tests/record_export.rs` proves the round trip, determinism, and every +verification finding. This document +is normative: the implementation follows it, not the other way around. + +An export never re-serializes a record. The stored canonical bytes of +each version (ADR 0011, ADR 0013) travel byte for byte, beside a +manifest that names them. Verifying an export needs nothing but the +export. + +## Vocabulary + +- **Unit** — one finalized version as exported: its canonical record + bytes and its unit manifest. +- **Archive** — the container holding one archive manifest and one or + more units. Every export, a single version included, is an archive, + so one reader and one verifier cover every scope. +- **Scope** — what the archive claims to contain: one version, one + record (every retained version, superseded originals included), one + enrollment (every finalized version of its records), or the whole + installation. + +## Container + +The container is a ZIP archive (APPNOTE 6.3): + +- every entry uses compression method 0 (stored); nothing is encrypted; + ZIP64 structures appear only when the container's size or entry count + requires them (a record is kilobytes, so no single entry ever does); +- entries are files only — no directory entries — named with forward + slashes and ASCII characters; +- entry order is the archive manifest first, then units in ascending + (`record_id`, `version_number`) order, `record.json` before + `manifest.json` within a unit; +- every entry's modification time is the export instant (DOS time, UTC + as written, so seconds round down to even), and every entry carries + Unix permissions `0644`; +- the archive holds nothing but what this document names; anything else + is a verification finding. + +Entry names: + +```text +manifest.json archive manifest +records/{record_id}/v{version_number}/record.json canonical record bytes +records/{record_id}/v{version_number}/manifest.json unit manifest +``` + +`{record_id}` and `{version_number}` are the decimal integers of the +unit's identity with no padding. The archive manifest states each +unit's `path` as the directory prefix (`records/12/v2`) and the path +must equal the derived form. + +## Determinism + +An archive is a pure function of its scope's stored rows and the +export instant: the same scope exported at the same instant produces +identical bytes. Manifests are canonical JSON under the record format's +JCS subset (ADR 0011): UTF-8, members sorted by code point, no +insignificant whitespace, integers only. Record bytes are copied from +storage unchanged. + +## `record.json` + +The version's `canonical_bytes` exactly as stored — a schema-1 or +schema-2 envelope under `jcs-v1`, presented under its own stored +`record_schema`. The export never upgrades, rewraps, or reformats them. +The envelope's own `record` and `instance` members are what the +verifier cross-checks against the manifests: the bytes commit to their +identity and lineage on their own (ADR 0011). + +## Unit manifest (`records/…/manifest.json`) + +```json +{ + "chain_hash": "…64 lowercase hex…", + "content_hash": "…64 lowercase hex…", + "exported_at": 1756753200, + "format": "consolebook-record-unit", + "format_version": 1, + "installation_id": "…", + "predecessor_content_hash": null, + "record_id": 12, + "record_schema": 2, + "version_number": 1 +} +``` + +| Member | Type | Meaning | +| --- | --- | --- | +| `format` | string | Always `consolebook-record-unit` | +| `format_version` | integer | `1`; bumped by any change to either manifest's shape | +| `installation_id` | string | The exporting installation's identity (`instance.installation_id`) | +| `record_id` | integer | The record's instance-local identity | +| `version_number` | integer | This version's number within the record (from 1) | +| `record_schema` | integer | The stored envelope schema of `record.json` | +| `content_hash` | string | The stored SHA-256 of `record.json`, lowercase hex | +| `chain_hash` | string | The stored integrity-chain hash (ADR 0011), lowercase hex | +| `predecessor_content_hash` | string or `null` | The prior version's content hash; `null` exactly when `version_number` is 1 | +| `exported_at` | integer | The export instant, UTC unix seconds; identical across the archive | + +A unit manifest repeats what the archive manifest says about the unit +so that a unit directory stands on its own; the two must agree. + +## Archive manifest (`manifest.json`) + +```json +{ + "exported_at": 1756753200, + "format": "consolebook-record-export", + "format_version": 1, + "installation_id": "…", + "scope": { "kind": "record", "record_id": 12 }, + "units": [ + { + "chain_hash": "…", + "content_hash": "…", + "path": "records/12/v1", + "predecessor_content_hash": null, + "record_id": 12, + "record_schema": 2, + "version_number": 1 + }, + { + "chain_hash": "…", + "content_hash": "…", + "path": "records/12/v2", + "predecessor_content_hash": "…", + "record_id": 12, + "record_schema": 2, + "version_number": 2 + } + ] +} +``` + +| Member | Type | Meaning | +| --- | --- | --- | +| `format` | string | Always `consolebook-record-export` | +| `format_version` | integer | `1` | +| `installation_id` | string | As in the unit manifest | +| `exported_at` | integer | The export instant, UTC unix seconds | +| `scope` | object | What the archive claims to contain (below) | +| `units` | array | Every unit, ascending by (`record_id`, `version_number`), no duplicates | + +`units[]` members are `path`, `record_id`, `version_number`, +`record_schema`, `content_hash`, `chain_hash`, and +`predecessor_content_hash`, with the unit-manifest meanings. + +### `scope` + +`kind` is one of: + +- `version` — with `record_id` and `version_number`; exactly one unit. +- `record` — with `record_id`; every retained version of that record. +- `enrollment` — with `enrollment_id`; every finalized version of every + record of that enrollment. +- `installation` — no further members; every finalized version the + installation holds. + +The scope is the exporter's statement of intent. Verification checks +the archive against itself; it cannot know whether an installation held +versions the archive omits. + +## Verification + +A verifier reads only the archive. It reports, per unit and for the +archive as a whole, and its verdict is `verified` only when every check +below passes. Wording stays honest (ADR 0010, ADR 0011, +`docs/records-integrity.md`): a verified export is internally +consistent with its stated fingerprints. Without the future signed +mode nothing in the archive proves which installation produced it, and +the checks are not tamper-proofing against whoever produced the file. + +Archive checks: + +1. the container is a readable ZIP archive; +2. `manifest.json` exists, parses, and carries a known `format` and + `format_version`; +3. `units` is ascending by (`record_id`, `version_number`) with no + duplicate identity, and each `path` equals the derived form; +4. every entry in the container is `manifest.json` or one of a listed + unit's two files — an unlisted entry is a finding; and +5. both files of every listed unit exist. + +Unit checks, for every listed unit: + +1. `manifest.json` parses, carries the known `format` and + `format_version`, and agrees with the archive entry and the archive + manifest on every shared member (`installation_id`, `exported_at`, + identity, schema, hashes); +2. `content_hash` equals SHA-256 over the bytes of `record.json`; +3. `record.json` parses as JSON and re-serializing it under the + canonical subset reproduces the identical bytes — the bytes are + canonical, so the hash is over the specified representation; +4. the envelope agrees with the manifest: `record.id`, + `record.version_number`, `record.record_schema`, + `record.predecessor_content_hash`, `instance`, and + `canonicalization` (`jcs-v1`); +5. `chain_hash` equals + `SHA-256("consolebook-version-v1" || 0x00 || predecessor || bytes)` + with `predecessor` the raw 32 bytes of `predecessor_content_hash`, + or 32 zero bytes when it is `null`; +6. `predecessor_content_hash` is `null` exactly when `version_number` + is 1; and +7. when the archive also lists (`record_id`, `version_number - 1`), + that unit's `content_hash` equals this unit's + `predecessor_content_hash` — reported as *linked*; a predecessor the + archive does not carry is reported as *not in export*, which is not + a failure (a single-version scope is legitimate), and a first + version reports *none*. + +`consolebook-server export verify ` runs exactly these checks, +prints one line per unit and every finding, and exits non-zero unless +the verdict is `verified`. It opens no data directory. + +## What the archive does not carry + +- **Drafts.** An unfinalized record is not a record; scopes contain + finalized versions only, and a scope with none is refused, never + exported empty. +- **Acknowledgments, amendments, and signoff history.** They are + separate records bound to versions; the trainee packet (#44 decision + 3) is the artifact that gathers them. An amendment's existence is + visible here only as a successor version's `predecessor_content_hash`. +- **Exporter identity.** Who exported what is the installation's + audit trail (`record_exported`), not the artifact's business. +- **Signatures.** The future signed mode adds them beside the hashes + without changing a record byte (`docs/records-integrity.md`). + +## Authorization (behavior of the producing installation) + +Export is a read in a portable shape, so it follows the read rules that +already exist rather than inventing parallel ones: a version or record +exports for whoever may read the record (workflow readers and the +trainee's own finalized record, ADR 0012); an enrollment exports for +whoever may read its training history; the whole installation exports +for holders of `export_records`. Every export is audited with actor and +subject and never with content. diff --git a/web/e2e/drafts.spec.ts b/web/e2e/drafts.spec.ts index 3a8741f..1bbe944 100644 --- a/web/e2e/drafts.spec.ts +++ b/web/e2e/drafts.spec.ts @@ -340,6 +340,19 @@ test('draft, collaborate, transfer, and submit a daily evaluation', async ({ pag await expect( page.getByText('Recomputed from the stored record: both fingerprints match.') ).toBeVisible(); + // The sealed record leaves as it was stored: the browser download is + // an archive whose record bytes are the canonical bytes verbatim, and + // the CLI verifies it from the file alone (ADR 0014). + const downloadPromise = page.waitForEvent('download'); + await page.getByRole('button', { name: 'Export this version' }).click(); + const download = await downloadPromise; + expect(download.suggestedFilename()).toMatch( + /^consolebook-record-\d+-v1-\d{8}T\d{6}Z\.zip$/ + ); + const archivePath = await download.path(); + const verified = await execFileAsync(BINARY, ['export', 'verify', archivePath]); + expect(verified.stdout).toContain('verified 1 of 1 units'); + await expect(page.getByText(/^Downloaded consolebook-record-/)).toBeVisible(); // The sealed record takes no further decisions or edits. await expect(page.getByRole('button', { name: 'Finalize record' })).toHaveCount(0); await expect(page.getByLabel('Most acceptable performance.')).toHaveCount(0); diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 3ed6f53..e4cdcd9 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -1182,3 +1182,68 @@ export function blankContent(name: string): VersionContent { } }; } + +// Record exports (Milestone 5 slice 1; docs/formats/record-export.md; +// ADR 0014): the stored canonical bytes travel verbatim beside +// manifests, and an archive verifies from its own contents alone. + +export interface ExportSummary { + installation_id: string; + record_count: number; + version_count: number; +} + +export function exportSummary(): Promise { + return request('/api/exports/summary'); +} + +/** Download path for one finalized version. */ +export function recordVersionExportPath(recordId: number, versionNumber: number): string { + return `/api/drafts/${recordId}/versions/${versionNumber}/export`; +} + +/** Download path for every retained version of a record. */ +export function recordExportPath(recordId: number): string { + return `/api/drafts/${recordId}/export`; +} + +/** Download path for every finalized version of an enrollment's records. */ +export function enrollmentExportPath(enrollmentId: number): string { + return `/api/enrollments/${enrollmentId}/export`; +} + +/** Download path for every finalized version the installation holds. */ +export function installationExportPath(): string { + return '/api/exports/records'; +} + +/** + * Fetches an export archive and hands it to the browser as a download, so + * a refusal surfaces as an error instead of a saved error document. + * Returns the server's file name. + */ +export async function downloadExport(path: string): Promise { + const response = await fetch(path); + if (!response.ok) { + let body: ApiErrorBody; + try { + body = (await response.json()) as ApiErrorBody; + } catch { + body = { error: 'unreachable', message: `server returned ${response.status}` }; + } + throw new ApiError(response.status, body); + } + const disposition = response.headers.get('Content-Disposition') ?? ''; + const named = /filename="([^"]+)"/.exec(disposition); + const fileName = named?.[1] ?? 'consolebook-export.zip'; + const url = URL.createObjectURL(await response.blob()); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = fileName; + document.body.append(anchor); + anchor.click(); + anchor.remove(); + // Revoke once the browser has had time to start the download. + setTimeout(() => URL.revokeObjectURL(url), 60_000); + return fileName; +} diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte index c9585fe..26480f6 100644 --- a/web/src/routes/+page.svelte +++ b/web/src/routes/+page.svelte @@ -6,8 +6,11 @@ createDraft, createUser, dailyForms, + downloadExport, + exportSummary, getHealth, getNotices, + installationExportPath, issueResetCode, logout, markNoticeRead, @@ -16,6 +19,7 @@ reviewQueue, type AssignedTrainee, type CreatedUser, + type ExportSummary, type Health, type MySession, type Notice, @@ -43,12 +47,16 @@ let canReview = $derived( data.session?.capabilities.includes('review_evaluation') ?? false ); + let canExport = $derived( + data.session?.capabilities.includes('export_records') ?? false + ); let health: Health | null = $state(null); let notices: Notice[] = $state([]); let assigned: AssignedTrainee[] = $state([]); let sessions: MySession[] = $state([]); let queue: ReviewQueueRow[] = $state([]); + let summary: ExportSummary | null = $state(null); $effect(() => { getHealth().then( (h) => (health = h), @@ -76,6 +84,12 @@ () => (queue = []) ); } + if (canExport) { + exportSummary().then( + (body) => (summary = body), + () => (summary = null) + ); + } }); async function acknowledge(notice: Notice) { @@ -194,6 +208,25 @@ busy = false; } } + + // The whole installation leaves as one archive: every finalized + // version as its stored bytes, verifiable from the archive alone + // (ADR 0014). This is the explicit export_records authority. + let exportError = $state(''); + let exported = $state(''); + async function exportInstallation() { + exportError = ''; + exported = ''; + busy = true; + try { + exported = await downloadExport(installationExportPath()); + } catch (err) { + exportError = + err instanceof ApiError ? err.message : 'the server could not be reached'; + } finally { + busy = false; + } + }

Installation status

@@ -239,6 +272,35 @@ {/if} +{#if canExport} +
+

Record exports

+ {#if summary === null} +

Checking finalized records…

+ {:else if summary.version_count === 0} +

No finalized records yet; there is nothing to export.

+ {:else} +

+ This installation holds {summary.version_count} finalized + {summary.version_count === 1 ? 'version' : 'versions'} across + {summary.record_count} + {summary.record_count === 1 ? 'record' : 'records'}. An export carries + every one as its stored canonical bytes beside a manifest and verifies + from the archive alone with consolebook-server export verify. +

+ + {#if exported} +

Downloaded {exported}.

+ {/if} + {/if} + {#if exportError} + + {/if} +
+{/if} + {#if canViewAssigned}

My trainees

diff --git a/web/src/routes/drafts/[id]/+page.svelte b/web/src/routes/drafts/[id]/+page.svelte index 824df5b..3f96228 100644 --- a/web/src/routes/drafts/[id]/+page.svelte +++ b/web/src/routes/drafts/[id]/+page.svelte @@ -5,12 +5,15 @@ acknowledgeRecord, amendRecord, attestRecord, + downloadExport, finalizeDraft, finalizedVersion, finalizedVersionAt, getAcknowledgment, getDraft, linkableDailies, + recordExportPath, + recordVersionExportPath, addSummaryLink, removeSummaryLink, reviewDraft, @@ -78,6 +81,30 @@ let verification: Verification | null = $state(null); let ack: Acknowledgment | null = $state(null); let versions: VersionHistoryRow[] = $state([]); + + // An export is the sealed record leaving as it was stored: the + // canonical bytes verbatim beside a manifest, verifiable anywhere + // (ADR 0014). + let exportError = $state(''); + let exported = $state(''); + async function exportArchive(path: string) { + exportError = ''; + exported = ''; + busy = true; + try { + exported = await downloadExport(path); + } catch (err) { + exportError = err instanceof ApiError ? err.message : 'the server could not be reached'; + } finally { + busy = false; + } + } + function exportSealedVersion() { + if (sealed === null) { + return; + } + void exportArchive(recordVersionExportPath(draftId, sealed.meta.version_number)); + } // The link picker for weekly summaries: the enrollment's finalized // dailies not yet linked. let linkable: SummaryLink[] = $state([]); @@ -915,6 +942,34 @@ stored; it is not by itself proof against a writer with direct database access.

+
+ + + {#if exported} + Downloaded {exported}. + {/if} + {#if exportError} + {exportError} + {/if} +
+

+ An export carries the stored record bytes verbatim beside a manifest + and verifies anywhere with consolebook-server export verify. +

{#if !viewingSuperseded} diff --git a/web/src/routes/enrollments/[id]/+page.svelte b/web/src/routes/enrollments/[id]/+page.svelte index a6444ae..46d802e 100644 --- a/web/src/routes/enrollments/[id]/+page.svelte +++ b/web/src/routes/enrollments/[id]/+page.svelte @@ -9,7 +9,9 @@ createDraft, createSession, dailyForms, + downloadExport, endAssignment, + enrollmentExportPath, getEnrollment, getProgramVersions, listSessions, @@ -50,6 +52,23 @@ let error = $state(''); let busy = $state(false); + // Every finalized version of this enrollment's records leaves as one + // archive of the stored record bytes with manifests (ADR 0014). + let exportError = $state(''); + let exported = $state(''); + async function exportEnrollment() { + exportError = ''; + exported = ''; + busy = true; + try { + exported = await downloadExport(enrollmentExportPath(enrollmentId)); + } catch (err) { + exportError = err instanceof ApiError ? err.message : 'the server could not be reached'; + } finally { + busy = false; + } + } + async function reload() { try { detail = await getEnrollment(enrollmentId); @@ -825,6 +844,27 @@ {/if} +
+

Export

+

+ Every finalized version of this enrollment's records — superseded + originals included — leaves as one archive of the stored record bytes + with manifests. Verify it anywhere with + consolebook-server export verify. +

+
+ + {#if exported} + Downloaded {exported}. + {/if} +
+ {#if exportError} + + {/if} +
+ {#if canAssign || canAuthor}

Weekly summaries

From 37efc28ec674569bb50e99cad5d9a618c27db96e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 19:29:56 +0000 Subject: [PATCH 2/7] records(export): check scope against units and reject duplicate entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two verifier gaps from review: the declared scope was copied into the report without checking the listed units against it, so a version scope listing another version, several units, or none at all still verified; and the zip reader indexes entries by name, so a name written twice — where extraction tools disagree on which copy they take — was invisible. - scope_findings: no_units, scope_cardinality (a version scope is exactly one unit), unit_outside_scope (version and record scopes); enrollment and installation scopes state nothing the bytes confirm. - duplicate_entry_findings walks the central directory itself (APPNOTE 6.3, ZIP64 fields included) and reports every name written more than once; an unreadable directory is its own finding. - docs/formats/record-export.md and ADR 0014 name both checks. - tests: empty units, mislabeled version scope, foreign record scope, and a hand-assembled container with a duplicated record entry whose genuine copy the reader resolves. Refs #45, #44 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- .../consolebook-server/src/record_export.rs | 174 +++++++++++++++++ .../consolebook-server/tests/record_export.rs | 177 ++++++++++++++++++ docs/decisions/0014-record-export-format.md | 11 +- docs/formats/record-export.md | 23 ++- 4 files changed, 374 insertions(+), 11 deletions(-) diff --git a/crates/consolebook-server/src/record_export.rs b/crates/consolebook-server/src/record_export.rs index 8a25404..1a7e0c7 100644 --- a/crates/consolebook-server/src/record_export.rs +++ b/crates/consolebook-server/src/record_export.rs @@ -529,6 +529,26 @@ pub enum Finding { }, /// Units are not strictly ascending by (record, version). UnitsOutOfOrder, + /// The manifest lists no unit; the format refuses empty exports. + NoUnits, + /// The declared scope calls for a different number of units. + ScopeCardinality { + expected: usize, + listed: usize, + }, + /// A listed unit's identity contradicts the declared scope. + UnitOutsideScope { + path: String, + }, + /// The container's central directory could not be walked. + CentralDirectoryUnreadable { + detail: String, + }, + /// The central directory names one entry more than once; extraction + /// tools disagree on which copy they take. + DuplicateEntry { + path: String, + }, UnitPathUnexpected { path: String, expected: String, @@ -586,6 +606,23 @@ impl fmt::Display for Finding { Self::UnitsOutOfOrder => { f.write_str("units are not strictly ascending by record and version") } + Self::NoUnits => f.write_str("the manifest lists no unit"), + Self::ScopeCardinality { expected, listed } => write!( + f, + "the declared scope calls for {expected} unit(s); the manifest lists {listed}" + ), + Self::UnitOutsideScope { path } => { + write!(f, "unit {path} is outside the declared scope") + } + Self::CentralDirectoryUnreadable { detail } => { + write!(f, "the central directory could not be walked: {detail}") + } + Self::DuplicateEntry { path } => { + write!( + f, + "entry {path} appears more than once in the central directory" + ) + } Self::UnitPathUnexpected { path, expected } => { write!(f, "unit path {path} should be {expected}") } @@ -719,6 +756,7 @@ pub fn verify_archive(bytes: &[u8]) -> ArchiveReport { } }; let names: Vec = archive.file_names().map(str::to_owned).collect(); + report.findings.extend(duplicate_entry_findings(bytes)); let manifest_bytes = match read_entry(&mut archive, ARCHIVE_MANIFEST_PATH) { Ok(Some(bytes)) => bytes, Ok(None) => { @@ -764,6 +802,7 @@ pub fn verify_archive(bytes: &[u8]) -> ArchiveReport { if !ordered { report.findings.push(Finding::UnitsOutOfOrder); } + report.findings.extend(scope_findings(&manifest)); let mut listed: BTreeSet = BTreeSet::new(); listed.insert(ARCHIVE_MANIFEST_PATH.to_owned()); for entry in &manifest.units { @@ -957,6 +996,141 @@ fn verify_unit( } } +/// The declared scope checked as far as the archive itself allows: no +/// scope is empty, a version scope is exactly its one unit, and a +/// record scope holds only that record's versions. Enrollment and +/// installation scopes state nothing the bytes can confirm. +fn scope_findings(manifest: &ArchiveManifest) -> Vec { + let mut findings = Vec::new(); + if manifest.units.is_empty() { + findings.push(Finding::NoUnits); + } + match manifest.scope { + Scope::Version { + record_id, + version_number, + } => { + if manifest.units.len() != 1 { + findings.push(Finding::ScopeCardinality { + expected: 1, + listed: manifest.units.len(), + }); + } + for entry in &manifest.units { + if (entry.record_id, entry.version_number) != (record_id, version_number) { + findings.push(Finding::UnitOutsideScope { + path: entry.path.clone(), + }); + } + } + } + Scope::Record { record_id } => { + for entry in &manifest.units { + if entry.record_id != record_id { + findings.push(Finding::UnitOutsideScope { + path: entry.path.clone(), + }); + } + } + } + Scope::Enrollment { .. } | Scope::Installation => {} + } + findings +} + +/// The reader keeps one entry per name; only the central directory +/// itself says whether a name was written twice. +fn duplicate_entry_findings(bytes: &[u8]) -> Vec { + match central_directory_names(bytes) { + Ok(directory) => { + let mut occurrences: BTreeMap<&str, usize> = BTreeMap::new(); + for name in &directory { + *occurrences.entry(name.as_str()).or_default() += 1; + } + occurrences + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(name, _)| Finding::DuplicateEntry { + path: name.to_owned(), + }) + .collect() + } + Err(detail) => vec![Finding::CentralDirectoryUnreadable { detail }], + } +} + +/// Every entry name in the central directory, duplicates included, in +/// directory order. The `zip` reader indexes entries by name and keeps +/// one per name, so a name written twice — which extraction tools +/// resolve differently — is visible only here. The walk follows +/// APPNOTE 6.3: the end-of-central-directory record (the last record, +/// followed by at most a 65535-byte comment), the ZIP64 locator and +/// record when the classic fields overflow, then the fixed 46-byte +/// central headers with their variable name, extra, and comment parts. +fn central_directory_names(bytes: &[u8]) -> std::result::Result, String> { + const EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06]; + const ZIP64_LOCATOR: [u8; 4] = [0x50, 0x4b, 0x06, 0x07]; + const ZIP64_EOCD: [u8; 4] = [0x50, 0x4b, 0x06, 0x06]; + const CENTRAL_HEADER: [u8; 4] = [0x50, 0x4b, 0x01, 0x02]; + let eocd = (0..=bytes.len().saturating_sub(22)) + .rev() + .take(usize::from(u16::MAX) + 1) + .find(|&at| bytes.get(at..at + 4) == Some(&EOCD[..])) + .ok_or("no end-of-central-directory record")?; + let mut count = u64::from(le_u16(bytes, eocd + 10)?); + let mut start = u64::from(le_u32(bytes, eocd + 16)?); + if count == u64::from(u16::MAX) || start == u64::from(u32::MAX) { + let locator = eocd + .checked_sub(20) + .filter(|&at| bytes.get(at..at + 4) == Some(&ZIP64_LOCATOR[..])) + .ok_or("ZIP64 fields without a ZIP64 locator")?; + let zip64 = usize::try_from(le_u64(bytes, locator + 8)?) + .map_err(|_| "ZIP64 record offset out of range".to_owned())?; + if bytes.get(zip64..zip64 + 4) != Some(&ZIP64_EOCD[..]) { + return Err("ZIP64 locator points at no ZIP64 record".to_owned()); + } + count = le_u64(bytes, zip64 + 32)?; + start = le_u64(bytes, zip64 + 48)?; + } + let mut at = usize::try_from(start).map_err(|_| "central directory offset out of range")?; + let mut names = Vec::new(); + for _ in 0..count { + if bytes.get(at..at + 4) != Some(&CENTRAL_HEADER[..]) { + return Err(format!("no central directory header at offset {at}")); + } + let name_len = usize::from(le_u16(bytes, at + 28)?); + let extra_len = usize::from(le_u16(bytes, at + 30)?); + let comment_len = usize::from(le_u16(bytes, at + 32)?); + let name = bytes + .get(at + 46..at + 46 + name_len) + .ok_or("truncated central directory header")?; + names.push(String::from_utf8_lossy(name).into_owned()); + at += 46 + name_len + extra_len + comment_len; + } + Ok(names) +} + +fn le_u16(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn le_u32(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 4) + .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn le_u64(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 8) + .map(|b| u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + /// Reads one entry: `Ok(None)` when absent, `Err(detail)` when the /// container cannot deliver it (a CRC mismatch included). fn read_entry( diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs index a54599d..7d69555 100644 --- a/crates/consolebook-server/tests/record_export.rs +++ b/crates/consolebook-server/tests/record_export.rs @@ -408,6 +408,85 @@ fn repack(entries: &[(String, Vec)]) -> Vec { writer.finish().expect("finish").into_inner() } +/// CRC-32 (IEEE), bitwise, for the hand-assembled container below. +fn crc32(bytes: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFF_u32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc +} + +fn le16(out: &mut Vec, value: usize) { + out.extend_from_slice(&u16::try_from(value).expect("fits u16").to_le_bytes()); +} + +fn le32(out: &mut Vec, value: usize) { + out.extend_from_slice(&u32::try_from(value).expect("fits u32").to_le_bytes()); +} + +/// A ZIP assembled by hand (APPNOTE 6.3, stored entries) so a name can +/// be written twice — the `zip` writer refuses to, and the `zip` reader +/// keeps one entry per name, which is exactly what the verifier must +/// see through. +fn handmade_zip(entries: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut central = Vec::new(); + for (name, data) in entries { + let offset = out.len(); + let crc = crc32(data); + // Local file header. + out.extend_from_slice(&[0x50, 0x4b, 0x03, 0x04]); + le16(&mut out, 20); // version needed + le16(&mut out, 0); // flags + le16(&mut out, 0); // stored + le16(&mut out, 0); // time + le16(&mut out, 0x21); // date: 1980-01-01 + out.extend_from_slice(&crc.to_le_bytes()); + le32(&mut out, data.len()); + le32(&mut out, data.len()); + le16(&mut out, name.len()); + le16(&mut out, 0); // extra + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(data); + // Central directory header. + central.extend_from_slice(&[0x50, 0x4b, 0x01, 0x02]); + le16(&mut central, 0x031E); // made by: Unix, 3.0 + le16(&mut central, 20); + le16(&mut central, 0); + le16(&mut central, 0); + le16(&mut central, 0); + le16(&mut central, 0x21); + central.extend_from_slice(&crc.to_le_bytes()); + le32(&mut central, data.len()); + le32(&mut central, data.len()); + le16(&mut central, name.len()); + le16(&mut central, 0); // extra + le16(&mut central, 0); // comment + le16(&mut central, 0); // disk + le16(&mut central, 0); // internal attributes + central.extend_from_slice(&(0o100_644_u32 << 16).to_le_bytes()); + le32(&mut central, offset); + central.extend_from_slice(name.as_bytes()); + } + let directory_offset = out.len(); + out.extend_from_slice(¢ral); + // End of central directory. + out.extend_from_slice(&[0x50, 0x4b, 0x05, 0x06]); + le16(&mut out, 0); + le16(&mut out, 0); + le16(&mut out, entries.len()); + le16(&mut out, entries.len()); + le32(&mut out, central.len()); + le32(&mut out, directory_offset); + le16(&mut out, 0); + out +} + fn edit_json(bytes: &[u8], edit: impl FnOnce(&mut serde_json::Value)) -> Vec { let mut value: serde_json::Value = serde_json::from_slice(bytes).expect("json"); edit(&mut value); @@ -824,6 +903,104 @@ async fn verification_names_every_finding() { ); assert_eq!(report.units[1].predecessor, PredecessorLink::None); + // The declared scope is checked against the listed units as far as + // the archive allows: no units at all, a version scope with two + // units (one of them outside it), a record scope naming another + // record. + let hollow = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"] = serde_json::Value::Array(Vec::new()); + })) + })); + let report = record_export::verify_archive(&hollow); + assert!(!report.verified()); + assert!(report.findings.contains(&Finding::NoUnits), "{report:?}"); + let mislabeled = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["scope"] = serde_json::json!({ + "kind": "version", + "record_id": s.record_id, + "version_number": 2, + }); + })) + })); + let report = record_export::verify_archive(&mislabeled); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::ScopeCardinality { + expected: 1, + listed: 2 + }), + "{report:?}" + ); + assert!( + report + .findings + .contains(&Finding::UnitOutsideScope { path: v1.clone() }), + "{report:?}" + ); + assert!( + !report + .findings + .contains(&Finding::UnitOutsideScope { path: v2.clone() }), + "{report:?}" + ); + let foreign = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["scope"]["record_id"] = serde_json::Value::from(9999); + })) + })); + let report = record_export::verify_archive(&foreign); + assert!(!report.verified()); + assert!( + report + .findings + .contains(&Finding::UnitOutsideScope { path: v1.clone() }), + "{report:?}" + ); + assert!( + report + .findings + .contains(&Finding::UnitOutsideScope { path: v2.clone() }), + "{report:?}" + ); + + // A name written twice: the reader resolves the genuine copy, so + // every per-unit check passes, and only the central directory shows + // the alternate record bytes another tool might extract instead. + let plain: Vec<(&str, &[u8])> = listed + .iter() + .map(|(name, content)| (name.as_str(), content.as_slice())) + .collect(); + let handmade = handmade_zip(&plain); + assert!( + record_export::verify_archive(&handmade).verified(), + "a hand-assembled container of the same entries verifies" + ); + let alternate = String::from_utf8(entry(&listed, &format!("{v2}/record.json")).to_vec()) + .expect("utf-8") + .replace("Corrected the invented", "Corrected the inveNted") + .into_bytes(); + let mut doubled: Vec<(&str, &[u8])> = Vec::new(); + for (name, content) in &plain { + if *name == format!("{v2}/record.json") { + doubled.push((name, alternate.as_slice())); + } + doubled.push((name, content)); + } + let report = record_export::verify_archive(&handmade_zip(&doubled)); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::DuplicateEntry { + path: format!("{v2}/record.json") + }), + "{report:?}" + ); + assert!( + report.units.iter().all(record_export::UnitReport::verified), + "the reader saw the genuine copy: {report:?}" + ); + // The untouched original still verifies after all that. assert!(record_export::verify_archive(&original).verified()); } diff --git a/docs/decisions/0014-record-export-format.md b/docs/decisions/0014-record-export-format.md index b310532..ccdbe62 100644 --- a/docs/decisions/0014-record-export-format.md +++ b/docs/decisions/0014-record-export-format.md @@ -53,10 +53,13 @@ may export, and what verification claims (#45; Milestone 5 slice 1). - The verifier recomputes the content hash over the bytes, proves the bytes are canonical by re-serialization, cross-checks the envelope's own `record` and `instance` members against the manifests, - recomputes the chain hash from the carried predecessor hash, and - links predecessors present in the same archive. It reports per unit - and per archive with typed findings; the verdict is `verified` only - when every check passes. + recomputes the chain hash from the carried predecessor hash, links + predecessors present in the same archive, checks the listed units + against the declared scope as far as the archive allows, and walks + the container's central directory itself so an entry name written + twice is a finding rather than whichever copy one reader happens to + pick. It reports per unit and per archive with typed findings; the + verdict is `verified` only when every check passes. - It ships as a library function and as `consolebook-server export verify `, which opens no data directory — the artifact is checked wherever it landed. diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md index 5be43ca..6356b98 100644 --- a/docs/formats/record-export.md +++ b/docs/formats/record-export.md @@ -163,8 +163,9 @@ so that a unit directory stands on its own; the two must agree. installation holds. The scope is the exporter's statement of intent. Verification checks -the archive against itself; it cannot know whether an installation held -versions the archive omits. +the archive against itself — a `version` scope must be exactly its one +unit, a `record` scope must hold only that record's versions — but it +cannot know whether an installation held versions the archive omits. ## Verification @@ -178,14 +179,22 @@ the checks are not tamper-proofing against whoever produced the file. Archive checks: -1. the container is a readable ZIP archive; +1. the container is a readable ZIP archive whose central directory names + each entry once — a name written twice is a finding, because + extraction tools disagree on which copy they take; 2. `manifest.json` exists, parses, and carries a known `format` and `format_version`; -3. `units` is ascending by (`record_id`, `version_number`) with no - duplicate identity, and each `path` equals the derived form; -4. every entry in the container is `manifest.json` or one of a listed +3. `units` lists at least one unit, ascending by (`record_id`, + `version_number`) with no duplicate identity, and each `path` equals + the derived form; +4. the listed units fit the declared scope as far as the archive can + tell: a `version` scope lists exactly one unit with that identity, a + `record` scope lists only units of that record; an `enrollment` or + `installation` scope states nothing the archive can confirm on its + own; +5. every entry in the container is `manifest.json` or one of a listed unit's two files — an unlisted entry is a finding; and -5. both files of every listed unit exist. +6. both files of every listed unit exist. Unit checks, for every listed unit: From b2571d0a393dca17d000475cb0fbdf8fefbd19b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 19:39:34 +0000 Subject: [PATCH 3/7] records(export): move verification into export_verify record_export crossed the 1,000-line reorganization threshold (AGENTS.md) once it owned both production and the independent check. Ownership now splits along the format's two sides: - record_export keeps the format vocabulary (constants, Scope, both manifests, unit_path, canonical_json) and production: authorization, row collection, the deterministic archive writer, audit, summary. - export_verify owns verification from the archive alone: findings, per-unit and per-archive reports, the unit and scope checks, the container walk including the central directory. No public behavior changes; the CLI, tests, and format document point at the new owner. Nothing persisted changes. Refs #45, #44 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- .../consolebook-server/src/export_verify.rs | 669 ++++++++++++++++++ crates/consolebook-server/src/lib.rs | 1 + crates/consolebook-server/src/main.rs | 4 +- .../consolebook-server/src/record_export.rs | 667 +---------------- .../consolebook-server/tests/record_export.rs | 61 +- docs/formats/record-export.md | 7 +- 6 files changed, 716 insertions(+), 693 deletions(-) create mode 100644 crates/consolebook-server/src/export_verify.rs diff --git a/crates/consolebook-server/src/export_verify.rs b/crates/consolebook-server/src/export_verify.rs new file mode 100644 index 0000000..163426b --- /dev/null +++ b/crates/consolebook-server/src/export_verify.rs @@ -0,0 +1,669 @@ +//! Verification of record export archives from their bytes alone +//! (ADR 0014; docs/formats/record-export.md; #45). +//! +//! `record_export` produces archives; this module owns the independent +//! check: typed findings, per-unit and per-archive reports, the +//! container walk (including the central directory, which the `zip` +//! reader collapses by name), and the normative check list of the +//! format document. The verdict is consistency with the stated +//! fingerprints, never tamper-proofing (ADR 0010, ADR 0011). Split out +//! of `record_export` when that module crossed the reorganization +//! threshold (AGENTS.md). + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::io::{Cursor, Read}; + +use serde::Serialize; +use serde_json::Value; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::canonical; +use crate::record_export::{ + ARCHIVE_FORMAT, ARCHIVE_MANIFEST_PATH, ArchiveManifest, FORMAT_VERSION, RECORD_FILE, Scope, + UNIT_FORMAT, UNIT_MANIFEST_FILE, UnitEntry, UnitManifest, canonical_json, unit_path, +}; + +/// One thing a verifier found wrong. The verdict derives from the +/// absence of findings; wording is presentation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Finding { + NotAnArchive { + detail: String, + }, + ArchiveManifestMissing, + ArchiveManifestUnreadable { + detail: String, + }, + UnsupportedFormat { + format: String, + format_version: i64, + }, + /// A manifest's bytes are not the canonical serialization of what + /// they parse to: a member is missing, reordered, or reformatted. + ManifestNotCanonical { + path: String, + }, + /// Units are not strictly ascending by (record, version). + UnitsOutOfOrder, + /// The manifest lists no unit; the format refuses empty exports. + NoUnits, + /// The declared scope calls for a different number of units. + ScopeCardinality { + expected: usize, + listed: usize, + }, + /// A listed unit's identity contradicts the declared scope. + UnitOutsideScope { + path: String, + }, + /// The container's central directory could not be walked. + CentralDirectoryUnreadable { + detail: String, + }, + /// The central directory names one entry more than once; extraction + /// tools disagree on which copy they take. + DuplicateEntry { + path: String, + }, + UnitPathUnexpected { + path: String, + expected: String, + }, + /// The container holds an entry the manifest does not name. + UnlistedEntry { + path: String, + }, + MissingEntry { + path: String, + }, + EntryUnreadable { + path: String, + detail: String, + }, + UnitManifestUnreadable { + detail: String, + }, + /// The unit manifest and the archive manifest disagree on a member. + UnitManifestDisagrees { + member: &'static str, + }, + ContentHashMismatch, + /// `record.json` is not canonical bytes (or not JSON at all). + NotCanonical { + detail: String, + }, + /// The envelope's own identity members disagree with the manifest. + EnvelopeDisagrees { + member: &'static str, + }, + ChainHashMismatch, + /// A first version with a predecessor, or a later one without. + LineageShape, + /// The predecessor is in the archive and its content hash is not + /// what this unit's chain was computed over. + PredecessorMismatch, +} + +impl fmt::Display for Finding { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAnArchive { detail } => write!(f, "not a readable ZIP archive: {detail}"), + Self::ArchiveManifestMissing => f.write_str("the archive manifest is missing"), + Self::ArchiveManifestUnreadable { detail } => { + write!(f, "the archive manifest is unreadable: {detail}") + } + Self::UnsupportedFormat { + format, + format_version, + } => write!(f, "unsupported format '{format}' version {format_version}"), + Self::ManifestNotCanonical { path } => { + write!(f, "{path} is not canonical JSON") + } + Self::UnitsOutOfOrder => { + f.write_str("units are not strictly ascending by record and version") + } + Self::NoUnits => f.write_str("the manifest lists no unit"), + Self::ScopeCardinality { expected, listed } => write!( + f, + "the declared scope calls for {expected} unit(s); the manifest lists {listed}" + ), + Self::UnitOutsideScope { path } => { + write!(f, "unit {path} is outside the declared scope") + } + Self::CentralDirectoryUnreadable { detail } => { + write!(f, "the central directory could not be walked: {detail}") + } + Self::DuplicateEntry { path } => { + write!( + f, + "entry {path} appears more than once in the central directory" + ) + } + Self::UnitPathUnexpected { path, expected } => { + write!(f, "unit path {path} should be {expected}") + } + Self::UnlistedEntry { path } => write!(f, "entry {path} is not listed by the manifest"), + Self::MissingEntry { path } => write!(f, "entry {path} is missing"), + Self::EntryUnreadable { path, detail } => { + write!(f, "entry {path} is unreadable: {detail}") + } + Self::UnitManifestUnreadable { detail } => { + write!(f, "the unit manifest is unreadable: {detail}") + } + Self::UnitManifestDisagrees { member } => { + write!( + f, + "the unit manifest disagrees with the archive on {member}" + ) + } + Self::ContentHashMismatch => { + f.write_str("the content hash does not match the record bytes") + } + Self::NotCanonical { detail } => { + write!(f, "the record bytes are not canonical: {detail}") + } + Self::EnvelopeDisagrees { member } => { + write!(f, "the record's own {member} disagrees with the manifest") + } + Self::ChainHashMismatch => { + f.write_str("the chain hash does not match the predecessor hash and record bytes") + } + Self::LineageShape => { + f.write_str("a predecessor hash is present exactly for versions after the first") + } + Self::PredecessorMismatch => { + f.write_str("the predecessor in this archive has a different content hash") + } + } + } +} + +/// Whether a unit's predecessor was checked against the archive. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PredecessorLink { + /// A first version. + None, + /// The predecessor is in the archive and its content hash matches. + Linked, + /// The archive does not carry the predecessor; the chain hash was + /// still recomputed from the carried predecessor hash. + NotInExport, +} + +impl fmt::Display for PredecessorLink { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::None => "none (first version)", + Self::Linked => "linked", + Self::NotInExport => "not in export", + }) + } +} + +/// One unit's verification. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct UnitReport { + pub path: String, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub predecessor: PredecessorLink, + pub findings: Vec, +} + +impl UnitReport { + #[must_use] + pub fn verified(&self) -> bool { + self.findings.is_empty() + } +} + +/// The whole archive's verification. `verified` when nothing was found +/// wrong anywhere: internally consistent with its stated fingerprints, +/// which is what the format can prove. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ArchiveReport { + pub installation_id: Option, + pub exported_at: Option, + pub scope: Option, + pub units: Vec, + pub findings: Vec, +} + +impl ArchiveReport { + #[must_use] + pub fn verified(&self) -> bool { + self.findings.is_empty() && self.units.iter().all(UnitReport::verified) + } + + /// The export instant as RFC 3339, for presentation. + #[must_use] + pub fn exported_at_rfc3339(&self) -> Option { + self.exported_at.and_then(|at| { + OffsetDateTime::from_unix_timestamp(at) + .ok()? + .format(&Rfc3339) + .ok() + }) + } +} + +type Archive<'a> = zip::ZipArchive>; + +/// Verifies an archive from its bytes alone, per the normative checks +/// in docs/formats/record-export.md. +#[must_use] +pub fn verify_archive(bytes: &[u8]) -> ArchiveReport { + let mut report = ArchiveReport { + installation_id: None, + exported_at: None, + scope: None, + units: Vec::new(), + findings: Vec::new(), + }; + let mut archive = match zip::ZipArchive::new(Cursor::new(bytes)) { + Ok(archive) => archive, + Err(err) => { + report.findings.push(Finding::NotAnArchive { + detail: err.to_string(), + }); + return report; + } + }; + let names: Vec = archive.file_names().map(str::to_owned).collect(); + report.findings.extend(duplicate_entry_findings(bytes)); + let manifest_bytes = match read_entry(&mut archive, ARCHIVE_MANIFEST_PATH) { + Ok(Some(bytes)) => bytes, + Ok(None) => { + report.findings.push(Finding::ArchiveManifestMissing); + return report; + } + Err(detail) => { + report.findings.push(Finding::EntryUnreadable { + path: ARCHIVE_MANIFEST_PATH.to_owned(), + detail, + }); + return report; + } + }; + let manifest: ArchiveManifest = match serde_json::from_slice(&manifest_bytes) { + Ok(manifest) => manifest, + Err(err) => { + report.findings.push(Finding::ArchiveManifestUnreadable { + detail: err.to_string(), + }); + return report; + } + }; + if manifest.format != ARCHIVE_FORMAT || manifest.format_version != FORMAT_VERSION { + report.findings.push(Finding::UnsupportedFormat { + format: manifest.format.clone(), + format_version: manifest.format_version, + }); + return report; + } + if canonical_json(&manifest).ok().as_deref() != Some(manifest_bytes.as_slice()) { + report.findings.push(Finding::ManifestNotCanonical { + path: ARCHIVE_MANIFEST_PATH.to_owned(), + }); + } + report.installation_id = Some(manifest.installation_id.clone()); + report.exported_at = Some(manifest.exported_at); + report.scope = Some(manifest.scope); + + let ordered = manifest.units.windows(2).all(|pair| { + (pair[0].record_id, pair[0].version_number) < (pair[1].record_id, pair[1].version_number) + }); + if !ordered { + report.findings.push(Finding::UnitsOutOfOrder); + } + report.findings.extend(scope_findings(&manifest)); + let mut listed: BTreeSet = BTreeSet::new(); + listed.insert(ARCHIVE_MANIFEST_PATH.to_owned()); + for entry in &manifest.units { + let expected = unit_path(entry.record_id, entry.version_number); + if entry.path != expected { + report.findings.push(Finding::UnitPathUnexpected { + path: entry.path.clone(), + expected, + }); + } + listed.insert(format!("{}/{RECORD_FILE}", entry.path)); + listed.insert(format!("{}/{UNIT_MANIFEST_FILE}", entry.path)); + } + for name in &names { + if !listed.contains(name) { + report + .findings + .push(Finding::UnlistedEntry { path: name.clone() }); + } + } + let by_identity: BTreeMap<(i64, i64), &UnitEntry> = manifest + .units + .iter() + .map(|entry| ((entry.record_id, entry.version_number), entry)) + .collect(); + for entry in &manifest.units { + report + .units + .push(verify_unit(&mut archive, &manifest, entry, &by_identity)); + } + report +} + +#[allow(clippy::too_many_lines)] +fn verify_unit( + archive: &mut Archive<'_>, + manifest: &ArchiveManifest, + entry: &UnitEntry, + by_identity: &BTreeMap<(i64, i64), &UnitEntry>, +) -> UnitReport { + let mut findings = Vec::new(); + let record_path = format!("{}/{RECORD_FILE}", entry.path); + let manifest_path = format!("{}/{UNIT_MANIFEST_FILE}", entry.path); + + match read_entry(archive, &manifest_path) { + Ok(Some(bytes)) => match serde_json::from_slice::(&bytes) { + Ok(unit) => { + if unit.format != UNIT_FORMAT || unit.format_version != FORMAT_VERSION { + findings.push(Finding::UnsupportedFormat { + format: unit.format.clone(), + format_version: unit.format_version, + }); + } + if canonical_json(&unit).ok().as_deref() != Some(bytes.as_slice()) { + findings.push(Finding::ManifestNotCanonical { + path: manifest_path.clone(), + }); + } + let disagreements: [(&'static str, bool); 8] = [ + ( + "installation_id", + unit.installation_id != manifest.installation_id, + ), + ("exported_at", unit.exported_at != manifest.exported_at), + ("record_id", unit.record_id != entry.record_id), + ( + "version_number", + unit.version_number != entry.version_number, + ), + ("record_schema", unit.record_schema != entry.record_schema), + ("content_hash", unit.content_hash != entry.content_hash), + ("chain_hash", unit.chain_hash != entry.chain_hash), + ( + "predecessor_content_hash", + unit.predecessor_content_hash != entry.predecessor_content_hash, + ), + ]; + for (member, disagrees) in disagreements { + if disagrees { + findings.push(Finding::UnitManifestDisagrees { member }); + } + } + } + Err(err) => findings.push(Finding::UnitManifestUnreadable { + detail: err.to_string(), + }), + }, + Ok(None) => findings.push(Finding::MissingEntry { + path: manifest_path.clone(), + }), + Err(detail) => findings.push(Finding::EntryUnreadable { + path: manifest_path.clone(), + detail, + }), + } + + match read_entry(archive, &record_path) { + Ok(Some(bytes)) => { + if canonical::content_hash_hex(&bytes) != entry.content_hash { + findings.push(Finding::ContentHashMismatch); + } + match serde_json::from_slice::(&bytes) { + Ok(envelope) => { + match canonical::canonical_bytes(&envelope) { + Ok(again) if again == bytes => {} + Ok(_) => findings.push(Finding::NotCanonical { + detail: "re-serialization differs from the stored bytes".to_owned(), + }), + Err(err) => findings.push(Finding::NotCanonical { + detail: err.to_string(), + }), + } + let predecessor = entry + .predecessor_content_hash + .as_ref() + .map_or(Value::Null, |hash| Value::String(hash.clone())); + let disagreements: [(&'static str, bool); 6] = [ + ("record.id", envelope["record"]["id"] != entry.record_id), + ( + "record.version_number", + envelope["record"]["version_number"] != entry.version_number, + ), + ( + "record.record_schema", + envelope["record"]["record_schema"] != entry.record_schema, + ), + ( + "record.predecessor_content_hash", + envelope["record"]["predecessor_content_hash"] != predecessor, + ), + ( + "instance", + envelope["instance"] != manifest.installation_id.as_str(), + ), + ( + "canonicalization", + envelope["canonicalization"] != canonical::CANONICALIZATION, + ), + ]; + for (member, disagrees) in disagreements { + if disagrees { + findings.push(Finding::EnvelopeDisagrees { member }); + } + } + } + Err(err) => findings.push(Finding::NotCanonical { + detail: format!("not JSON: {err}"), + }), + } + match canonical::chain_hash_hex(entry.predecessor_content_hash.as_deref(), &bytes) { + Ok(chain) if chain == entry.chain_hash => {} + _ => findings.push(Finding::ChainHashMismatch), + } + } + Ok(None) => findings.push(Finding::MissingEntry { + path: record_path.clone(), + }), + Err(detail) => findings.push(Finding::EntryUnreadable { + path: record_path.clone(), + detail, + }), + } + + if (entry.version_number == 1) != entry.predecessor_content_hash.is_none() { + findings.push(Finding::LineageShape); + } + let predecessor = match &entry.predecessor_content_hash { + None => PredecessorLink::None, + Some(hash) => match entry + .version_number + .checked_sub(1) + .and_then(|number| by_identity.get(&(entry.record_id, number))) + { + Some(previous) => { + if previous.content_hash != *hash { + findings.push(Finding::PredecessorMismatch); + } + PredecessorLink::Linked + } + None => PredecessorLink::NotInExport, + }, + }; + + UnitReport { + path: entry.path.clone(), + record_id: entry.record_id, + version_number: entry.version_number, + record_schema: entry.record_schema, + predecessor, + findings, + } +} + +/// The declared scope checked as far as the archive itself allows: no +/// scope is empty, a version scope is exactly its one unit, and a +/// record scope holds only that record's versions. Enrollment and +/// installation scopes state nothing the bytes can confirm. +fn scope_findings(manifest: &ArchiveManifest) -> Vec { + let mut findings = Vec::new(); + if manifest.units.is_empty() { + findings.push(Finding::NoUnits); + } + match manifest.scope { + Scope::Version { + record_id, + version_number, + } => { + if manifest.units.len() != 1 { + findings.push(Finding::ScopeCardinality { + expected: 1, + listed: manifest.units.len(), + }); + } + for entry in &manifest.units { + if (entry.record_id, entry.version_number) != (record_id, version_number) { + findings.push(Finding::UnitOutsideScope { + path: entry.path.clone(), + }); + } + } + } + Scope::Record { record_id } => { + for entry in &manifest.units { + if entry.record_id != record_id { + findings.push(Finding::UnitOutsideScope { + path: entry.path.clone(), + }); + } + } + } + Scope::Enrollment { .. } | Scope::Installation => {} + } + findings +} + +/// The reader keeps one entry per name; only the central directory +/// itself says whether a name was written twice. +fn duplicate_entry_findings(bytes: &[u8]) -> Vec { + match central_directory_names(bytes) { + Ok(directory) => { + let mut occurrences: BTreeMap<&str, usize> = BTreeMap::new(); + for name in &directory { + *occurrences.entry(name.as_str()).or_default() += 1; + } + occurrences + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(name, _)| Finding::DuplicateEntry { + path: name.to_owned(), + }) + .collect() + } + Err(detail) => vec![Finding::CentralDirectoryUnreadable { detail }], + } +} + +/// Every entry name in the central directory, duplicates included, in +/// directory order. The `zip` reader indexes entries by name and keeps +/// one per name, so a name written twice — which extraction tools +/// resolve differently — is visible only here. The walk follows +/// APPNOTE 6.3: the end-of-central-directory record (the last record, +/// followed by at most a 65535-byte comment), the ZIP64 locator and +/// record when the classic fields overflow, then the fixed 46-byte +/// central headers with their variable name, extra, and comment parts. +fn central_directory_names(bytes: &[u8]) -> std::result::Result, String> { + const EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06]; + const ZIP64_LOCATOR: [u8; 4] = [0x50, 0x4b, 0x06, 0x07]; + const ZIP64_EOCD: [u8; 4] = [0x50, 0x4b, 0x06, 0x06]; + const CENTRAL_HEADER: [u8; 4] = [0x50, 0x4b, 0x01, 0x02]; + let eocd = (0..=bytes.len().saturating_sub(22)) + .rev() + .take(usize::from(u16::MAX) + 1) + .find(|&at| bytes.get(at..at + 4) == Some(&EOCD[..])) + .ok_or("no end-of-central-directory record")?; + let mut count = u64::from(le_u16(bytes, eocd + 10)?); + let mut start = u64::from(le_u32(bytes, eocd + 16)?); + if count == u64::from(u16::MAX) || start == u64::from(u32::MAX) { + let locator = eocd + .checked_sub(20) + .filter(|&at| bytes.get(at..at + 4) == Some(&ZIP64_LOCATOR[..])) + .ok_or("ZIP64 fields without a ZIP64 locator")?; + let zip64 = usize::try_from(le_u64(bytes, locator + 8)?) + .map_err(|_| "ZIP64 record offset out of range".to_owned())?; + if bytes.get(zip64..zip64 + 4) != Some(&ZIP64_EOCD[..]) { + return Err("ZIP64 locator points at no ZIP64 record".to_owned()); + } + count = le_u64(bytes, zip64 + 32)?; + start = le_u64(bytes, zip64 + 48)?; + } + let mut at = usize::try_from(start).map_err(|_| "central directory offset out of range")?; + let mut names = Vec::new(); + for _ in 0..count { + if bytes.get(at..at + 4) != Some(&CENTRAL_HEADER[..]) { + return Err(format!("no central directory header at offset {at}")); + } + let name_len = usize::from(le_u16(bytes, at + 28)?); + let extra_len = usize::from(le_u16(bytes, at + 30)?); + let comment_len = usize::from(le_u16(bytes, at + 32)?); + let name = bytes + .get(at + 46..at + 46 + name_len) + .ok_or("truncated central directory header")?; + names.push(String::from_utf8_lossy(name).into_owned()); + at += 46 + name_len + extra_len + comment_len; + } + Ok(names) +} + +fn le_u16(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn le_u32(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 4) + .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn le_u64(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 8) + .map(|b| u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +/// Reads one entry: `Ok(None)` when absent, `Err(detail)` when the +/// container cannot deliver it (a CRC mismatch included). +fn read_entry( + archive: &mut Archive<'_>, + name: &str, +) -> std::result::Result>, String> { + match archive.by_name(name) { + Ok(mut file) => { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes) + .map_err(|err| err.to_string())?; + Ok(Some(bytes)) + } + Err(zip::result::ZipError::FileNotFound) => Ok(None), + Err(err) => Err(err.to_string()), + } +} diff --git a/crates/consolebook-server/src/lib.rs b/crates/consolebook-server/src/lib.rs index 7fc50a8..881a3e8 100644 --- a/crates/consolebook-server/src/lib.rs +++ b/crates/consolebook-server/src/lib.rs @@ -18,6 +18,7 @@ pub mod draft_review; pub mod drafts_http; pub mod enrollments; pub mod evaluation_drafts; +pub mod export_verify; pub mod exports_http; pub mod finalization; pub mod http; diff --git a/crates/consolebook-server/src/main.rs b/crates/consolebook-server/src/main.rs index c23edf3..d9378f1 100644 --- a/crates/consolebook-server/src/main.rs +++ b/crates/consolebook-server/src/main.rs @@ -12,7 +12,7 @@ use consolebook_server::doctor::Verdict; use consolebook_server::serve_lock::ServeLock; use consolebook_server::users::{IssueRefusal, ResetOrigin}; use consolebook_server::{ - VERSION, backup, doctor, http, record_export, restore, scheduler, setup, storage, users, + VERSION, backup, doctor, export_verify, http, restore, scheduler, setup, storage, users, }; #[derive(Parser)] @@ -298,7 +298,7 @@ async fn run_restore(data_dir: &std::path::Path, snapshot: &std::path::Path) -> fn run_export_verify(archive: &std::path::Path) -> Result { let bytes = std::fs::read(archive).with_context(|| format!("reading {}", archive.display()))?; - let report = record_export::verify_archive(&bytes); + let report = export_verify::verify_archive(&bytes); if let Some(id) = &report.installation_id { println!("installation {id}"); } diff --git a/crates/consolebook-server/src/record_export.rs b/crates/consolebook-server/src/record_export.rs index 1a7e0c7..c340e16 100644 --- a/crates/consolebook-server/src/record_export.rs +++ b/crates/consolebook-server/src/record_export.rs @@ -1,28 +1,24 @@ -//! Structured record exports (ADR 0014; docs/formats/record-export.md; -//! #45). +//! Structured record exports: the format vocabulary and the producing +//! side (ADR 0014; docs/formats/record-export.md; #45). //! //! An export unit is one finalized version's stored canonical bytes, //! copied verbatim, beside a canonical-JSON unit manifest; an archive //! is a ZIP container with stored entries in a fixed order holding an //! archive manifest and its units. The archive is a pure function of //! the scope's stored rows and the export instant, so the same scope -//! exported at the same instant is byte-identical. Verification reads -//! only the archive and reports typed findings; its verdict is -//! consistency with the stated fingerprints, never tamper-proofing -//! (ADR 0010, ADR 0011). Export follows the read rules that already -//! exist — a unit contains exactly what its reader may already read — -//! and the installation scope takes `export_records`. +//! exported at the same instant is byte-identical. Export follows the +//! read rules that already exist — a unit contains exactly what its +//! reader may already read — and the installation scope takes +//! `export_records`. Verification from the archive alone is +//! `export_verify`'s, which reads the manifests defined here. -use std::collections::{BTreeMap, BTreeSet}; use std::fmt; -use std::io::{Cursor, Read, Write}; +use std::io::{Cursor, Write}; use anyhow::{Context, Result, anyhow}; use serde::{Deserialize, Serialize}; -use serde_json::Value; use sqlx::{Row, SqlitePool}; use time::OffsetDateTime; -use time::format_description::well_known::Rfc3339; use zip::CompressionMethod; use zip::write::{SimpleFileOptions, ZipWriter}; @@ -467,7 +463,7 @@ fn add_entry( /// Manifests are canonical JSON under the record format's subset, so /// the archive is deterministic and a manifest is itself checkable. -fn canonical_json(value: &T) -> Result> { +pub(crate) fn canonical_json(value: &T) -> Result> { let value = serde_json::to_value(value).context("serializing manifest")?; canonical::canonical_bytes(&value) } @@ -503,648 +499,3 @@ fn file_name(scope: Scope, exported_at: i64) -> Result { }; Ok(format!("consolebook-{scope_part}-{stamp}.zip")) } - -// ------------------------------------------------------------ verifying - -/// One thing a verifier found wrong. The verdict derives from the -/// absence of findings; wording is presentation. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum Finding { - NotAnArchive { - detail: String, - }, - ArchiveManifestMissing, - ArchiveManifestUnreadable { - detail: String, - }, - UnsupportedFormat { - format: String, - format_version: i64, - }, - /// A manifest's bytes are not the canonical serialization of what - /// they parse to: a member is missing, reordered, or reformatted. - ManifestNotCanonical { - path: String, - }, - /// Units are not strictly ascending by (record, version). - UnitsOutOfOrder, - /// The manifest lists no unit; the format refuses empty exports. - NoUnits, - /// The declared scope calls for a different number of units. - ScopeCardinality { - expected: usize, - listed: usize, - }, - /// A listed unit's identity contradicts the declared scope. - UnitOutsideScope { - path: String, - }, - /// The container's central directory could not be walked. - CentralDirectoryUnreadable { - detail: String, - }, - /// The central directory names one entry more than once; extraction - /// tools disagree on which copy they take. - DuplicateEntry { - path: String, - }, - UnitPathUnexpected { - path: String, - expected: String, - }, - /// The container holds an entry the manifest does not name. - UnlistedEntry { - path: String, - }, - MissingEntry { - path: String, - }, - EntryUnreadable { - path: String, - detail: String, - }, - UnitManifestUnreadable { - detail: String, - }, - /// The unit manifest and the archive manifest disagree on a member. - UnitManifestDisagrees { - member: &'static str, - }, - ContentHashMismatch, - /// `record.json` is not canonical bytes (or not JSON at all). - NotCanonical { - detail: String, - }, - /// The envelope's own identity members disagree with the manifest. - EnvelopeDisagrees { - member: &'static str, - }, - ChainHashMismatch, - /// A first version with a predecessor, or a later one without. - LineageShape, - /// The predecessor is in the archive and its content hash is not - /// what this unit's chain was computed over. - PredecessorMismatch, -} - -impl fmt::Display for Finding { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::NotAnArchive { detail } => write!(f, "not a readable ZIP archive: {detail}"), - Self::ArchiveManifestMissing => f.write_str("the archive manifest is missing"), - Self::ArchiveManifestUnreadable { detail } => { - write!(f, "the archive manifest is unreadable: {detail}") - } - Self::UnsupportedFormat { - format, - format_version, - } => write!(f, "unsupported format '{format}' version {format_version}"), - Self::ManifestNotCanonical { path } => { - write!(f, "{path} is not canonical JSON") - } - Self::UnitsOutOfOrder => { - f.write_str("units are not strictly ascending by record and version") - } - Self::NoUnits => f.write_str("the manifest lists no unit"), - Self::ScopeCardinality { expected, listed } => write!( - f, - "the declared scope calls for {expected} unit(s); the manifest lists {listed}" - ), - Self::UnitOutsideScope { path } => { - write!(f, "unit {path} is outside the declared scope") - } - Self::CentralDirectoryUnreadable { detail } => { - write!(f, "the central directory could not be walked: {detail}") - } - Self::DuplicateEntry { path } => { - write!( - f, - "entry {path} appears more than once in the central directory" - ) - } - Self::UnitPathUnexpected { path, expected } => { - write!(f, "unit path {path} should be {expected}") - } - Self::UnlistedEntry { path } => write!(f, "entry {path} is not listed by the manifest"), - Self::MissingEntry { path } => write!(f, "entry {path} is missing"), - Self::EntryUnreadable { path, detail } => { - write!(f, "entry {path} is unreadable: {detail}") - } - Self::UnitManifestUnreadable { detail } => { - write!(f, "the unit manifest is unreadable: {detail}") - } - Self::UnitManifestDisagrees { member } => { - write!( - f, - "the unit manifest disagrees with the archive on {member}" - ) - } - Self::ContentHashMismatch => { - f.write_str("the content hash does not match the record bytes") - } - Self::NotCanonical { detail } => { - write!(f, "the record bytes are not canonical: {detail}") - } - Self::EnvelopeDisagrees { member } => { - write!(f, "the record's own {member} disagrees with the manifest") - } - Self::ChainHashMismatch => { - f.write_str("the chain hash does not match the predecessor hash and record bytes") - } - Self::LineageShape => { - f.write_str("a predecessor hash is present exactly for versions after the first") - } - Self::PredecessorMismatch => { - f.write_str("the predecessor in this archive has a different content hash") - } - } - } -} - -/// Whether a unit's predecessor was checked against the archive. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum PredecessorLink { - /// A first version. - None, - /// The predecessor is in the archive and its content hash matches. - Linked, - /// The archive does not carry the predecessor; the chain hash was - /// still recomputed from the carried predecessor hash. - NotInExport, -} - -impl fmt::Display for PredecessorLink { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::None => "none (first version)", - Self::Linked => "linked", - Self::NotInExport => "not in export", - }) - } -} - -/// One unit's verification. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct UnitReport { - pub path: String, - pub record_id: i64, - pub version_number: i64, - pub record_schema: i64, - pub predecessor: PredecessorLink, - pub findings: Vec, -} - -impl UnitReport { - #[must_use] - pub fn verified(&self) -> bool { - self.findings.is_empty() - } -} - -/// The whole archive's verification. `verified` when nothing was found -/// wrong anywhere: internally consistent with its stated fingerprints, -/// which is what the format can prove. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct ArchiveReport { - pub installation_id: Option, - pub exported_at: Option, - pub scope: Option, - pub units: Vec, - pub findings: Vec, -} - -impl ArchiveReport { - #[must_use] - pub fn verified(&self) -> bool { - self.findings.is_empty() && self.units.iter().all(UnitReport::verified) - } - - /// The export instant as RFC 3339, for presentation. - #[must_use] - pub fn exported_at_rfc3339(&self) -> Option { - self.exported_at.and_then(|at| { - OffsetDateTime::from_unix_timestamp(at) - .ok()? - .format(&Rfc3339) - .ok() - }) - } -} - -type Archive<'a> = zip::ZipArchive>; - -/// Verifies an archive from its bytes alone, per the normative checks -/// in docs/formats/record-export.md. -#[must_use] -pub fn verify_archive(bytes: &[u8]) -> ArchiveReport { - let mut report = ArchiveReport { - installation_id: None, - exported_at: None, - scope: None, - units: Vec::new(), - findings: Vec::new(), - }; - let mut archive = match zip::ZipArchive::new(Cursor::new(bytes)) { - Ok(archive) => archive, - Err(err) => { - report.findings.push(Finding::NotAnArchive { - detail: err.to_string(), - }); - return report; - } - }; - let names: Vec = archive.file_names().map(str::to_owned).collect(); - report.findings.extend(duplicate_entry_findings(bytes)); - let manifest_bytes = match read_entry(&mut archive, ARCHIVE_MANIFEST_PATH) { - Ok(Some(bytes)) => bytes, - Ok(None) => { - report.findings.push(Finding::ArchiveManifestMissing); - return report; - } - Err(detail) => { - report.findings.push(Finding::EntryUnreadable { - path: ARCHIVE_MANIFEST_PATH.to_owned(), - detail, - }); - return report; - } - }; - let manifest: ArchiveManifest = match serde_json::from_slice(&manifest_bytes) { - Ok(manifest) => manifest, - Err(err) => { - report.findings.push(Finding::ArchiveManifestUnreadable { - detail: err.to_string(), - }); - return report; - } - }; - if manifest.format != ARCHIVE_FORMAT || manifest.format_version != FORMAT_VERSION { - report.findings.push(Finding::UnsupportedFormat { - format: manifest.format.clone(), - format_version: manifest.format_version, - }); - return report; - } - if canonical_json(&manifest).ok().as_deref() != Some(manifest_bytes.as_slice()) { - report.findings.push(Finding::ManifestNotCanonical { - path: ARCHIVE_MANIFEST_PATH.to_owned(), - }); - } - report.installation_id = Some(manifest.installation_id.clone()); - report.exported_at = Some(manifest.exported_at); - report.scope = Some(manifest.scope); - - let ordered = manifest.units.windows(2).all(|pair| { - (pair[0].record_id, pair[0].version_number) < (pair[1].record_id, pair[1].version_number) - }); - if !ordered { - report.findings.push(Finding::UnitsOutOfOrder); - } - report.findings.extend(scope_findings(&manifest)); - let mut listed: BTreeSet = BTreeSet::new(); - listed.insert(ARCHIVE_MANIFEST_PATH.to_owned()); - for entry in &manifest.units { - let expected = unit_path(entry.record_id, entry.version_number); - if entry.path != expected { - report.findings.push(Finding::UnitPathUnexpected { - path: entry.path.clone(), - expected, - }); - } - listed.insert(format!("{}/{RECORD_FILE}", entry.path)); - listed.insert(format!("{}/{UNIT_MANIFEST_FILE}", entry.path)); - } - for name in &names { - if !listed.contains(name) { - report - .findings - .push(Finding::UnlistedEntry { path: name.clone() }); - } - } - let by_identity: BTreeMap<(i64, i64), &UnitEntry> = manifest - .units - .iter() - .map(|entry| ((entry.record_id, entry.version_number), entry)) - .collect(); - for entry in &manifest.units { - report - .units - .push(verify_unit(&mut archive, &manifest, entry, &by_identity)); - } - report -} - -#[allow(clippy::too_many_lines)] -fn verify_unit( - archive: &mut Archive<'_>, - manifest: &ArchiveManifest, - entry: &UnitEntry, - by_identity: &BTreeMap<(i64, i64), &UnitEntry>, -) -> UnitReport { - let mut findings = Vec::new(); - let record_path = format!("{}/{RECORD_FILE}", entry.path); - let manifest_path = format!("{}/{UNIT_MANIFEST_FILE}", entry.path); - - match read_entry(archive, &manifest_path) { - Ok(Some(bytes)) => match serde_json::from_slice::(&bytes) { - Ok(unit) => { - if unit.format != UNIT_FORMAT || unit.format_version != FORMAT_VERSION { - findings.push(Finding::UnsupportedFormat { - format: unit.format.clone(), - format_version: unit.format_version, - }); - } - if canonical_json(&unit).ok().as_deref() != Some(bytes.as_slice()) { - findings.push(Finding::ManifestNotCanonical { - path: manifest_path.clone(), - }); - } - let disagreements: [(&'static str, bool); 8] = [ - ( - "installation_id", - unit.installation_id != manifest.installation_id, - ), - ("exported_at", unit.exported_at != manifest.exported_at), - ("record_id", unit.record_id != entry.record_id), - ( - "version_number", - unit.version_number != entry.version_number, - ), - ("record_schema", unit.record_schema != entry.record_schema), - ("content_hash", unit.content_hash != entry.content_hash), - ("chain_hash", unit.chain_hash != entry.chain_hash), - ( - "predecessor_content_hash", - unit.predecessor_content_hash != entry.predecessor_content_hash, - ), - ]; - for (member, disagrees) in disagreements { - if disagrees { - findings.push(Finding::UnitManifestDisagrees { member }); - } - } - } - Err(err) => findings.push(Finding::UnitManifestUnreadable { - detail: err.to_string(), - }), - }, - Ok(None) => findings.push(Finding::MissingEntry { - path: manifest_path.clone(), - }), - Err(detail) => findings.push(Finding::EntryUnreadable { - path: manifest_path.clone(), - detail, - }), - } - - match read_entry(archive, &record_path) { - Ok(Some(bytes)) => { - if canonical::content_hash_hex(&bytes) != entry.content_hash { - findings.push(Finding::ContentHashMismatch); - } - match serde_json::from_slice::(&bytes) { - Ok(envelope) => { - match canonical::canonical_bytes(&envelope) { - Ok(again) if again == bytes => {} - Ok(_) => findings.push(Finding::NotCanonical { - detail: "re-serialization differs from the stored bytes".to_owned(), - }), - Err(err) => findings.push(Finding::NotCanonical { - detail: err.to_string(), - }), - } - let predecessor = entry - .predecessor_content_hash - .as_ref() - .map_or(Value::Null, |hash| Value::String(hash.clone())); - let disagreements: [(&'static str, bool); 6] = [ - ("record.id", envelope["record"]["id"] != entry.record_id), - ( - "record.version_number", - envelope["record"]["version_number"] != entry.version_number, - ), - ( - "record.record_schema", - envelope["record"]["record_schema"] != entry.record_schema, - ), - ( - "record.predecessor_content_hash", - envelope["record"]["predecessor_content_hash"] != predecessor, - ), - ( - "instance", - envelope["instance"] != manifest.installation_id.as_str(), - ), - ( - "canonicalization", - envelope["canonicalization"] != canonical::CANONICALIZATION, - ), - ]; - for (member, disagrees) in disagreements { - if disagrees { - findings.push(Finding::EnvelopeDisagrees { member }); - } - } - } - Err(err) => findings.push(Finding::NotCanonical { - detail: format!("not JSON: {err}"), - }), - } - match canonical::chain_hash_hex(entry.predecessor_content_hash.as_deref(), &bytes) { - Ok(chain) if chain == entry.chain_hash => {} - _ => findings.push(Finding::ChainHashMismatch), - } - } - Ok(None) => findings.push(Finding::MissingEntry { - path: record_path.clone(), - }), - Err(detail) => findings.push(Finding::EntryUnreadable { - path: record_path.clone(), - detail, - }), - } - - if (entry.version_number == 1) != entry.predecessor_content_hash.is_none() { - findings.push(Finding::LineageShape); - } - let predecessor = match &entry.predecessor_content_hash { - None => PredecessorLink::None, - Some(hash) => match entry - .version_number - .checked_sub(1) - .and_then(|number| by_identity.get(&(entry.record_id, number))) - { - Some(previous) => { - if previous.content_hash != *hash { - findings.push(Finding::PredecessorMismatch); - } - PredecessorLink::Linked - } - None => PredecessorLink::NotInExport, - }, - }; - - UnitReport { - path: entry.path.clone(), - record_id: entry.record_id, - version_number: entry.version_number, - record_schema: entry.record_schema, - predecessor, - findings, - } -} - -/// The declared scope checked as far as the archive itself allows: no -/// scope is empty, a version scope is exactly its one unit, and a -/// record scope holds only that record's versions. Enrollment and -/// installation scopes state nothing the bytes can confirm. -fn scope_findings(manifest: &ArchiveManifest) -> Vec { - let mut findings = Vec::new(); - if manifest.units.is_empty() { - findings.push(Finding::NoUnits); - } - match manifest.scope { - Scope::Version { - record_id, - version_number, - } => { - if manifest.units.len() != 1 { - findings.push(Finding::ScopeCardinality { - expected: 1, - listed: manifest.units.len(), - }); - } - for entry in &manifest.units { - if (entry.record_id, entry.version_number) != (record_id, version_number) { - findings.push(Finding::UnitOutsideScope { - path: entry.path.clone(), - }); - } - } - } - Scope::Record { record_id } => { - for entry in &manifest.units { - if entry.record_id != record_id { - findings.push(Finding::UnitOutsideScope { - path: entry.path.clone(), - }); - } - } - } - Scope::Enrollment { .. } | Scope::Installation => {} - } - findings -} - -/// The reader keeps one entry per name; only the central directory -/// itself says whether a name was written twice. -fn duplicate_entry_findings(bytes: &[u8]) -> Vec { - match central_directory_names(bytes) { - Ok(directory) => { - let mut occurrences: BTreeMap<&str, usize> = BTreeMap::new(); - for name in &directory { - *occurrences.entry(name.as_str()).or_default() += 1; - } - occurrences - .into_iter() - .filter(|(_, count)| *count > 1) - .map(|(name, _)| Finding::DuplicateEntry { - path: name.to_owned(), - }) - .collect() - } - Err(detail) => vec![Finding::CentralDirectoryUnreadable { detail }], - } -} - -/// Every entry name in the central directory, duplicates included, in -/// directory order. The `zip` reader indexes entries by name and keeps -/// one per name, so a name written twice — which extraction tools -/// resolve differently — is visible only here. The walk follows -/// APPNOTE 6.3: the end-of-central-directory record (the last record, -/// followed by at most a 65535-byte comment), the ZIP64 locator and -/// record when the classic fields overflow, then the fixed 46-byte -/// central headers with their variable name, extra, and comment parts. -fn central_directory_names(bytes: &[u8]) -> std::result::Result, String> { - const EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06]; - const ZIP64_LOCATOR: [u8; 4] = [0x50, 0x4b, 0x06, 0x07]; - const ZIP64_EOCD: [u8; 4] = [0x50, 0x4b, 0x06, 0x06]; - const CENTRAL_HEADER: [u8; 4] = [0x50, 0x4b, 0x01, 0x02]; - let eocd = (0..=bytes.len().saturating_sub(22)) - .rev() - .take(usize::from(u16::MAX) + 1) - .find(|&at| bytes.get(at..at + 4) == Some(&EOCD[..])) - .ok_or("no end-of-central-directory record")?; - let mut count = u64::from(le_u16(bytes, eocd + 10)?); - let mut start = u64::from(le_u32(bytes, eocd + 16)?); - if count == u64::from(u16::MAX) || start == u64::from(u32::MAX) { - let locator = eocd - .checked_sub(20) - .filter(|&at| bytes.get(at..at + 4) == Some(&ZIP64_LOCATOR[..])) - .ok_or("ZIP64 fields without a ZIP64 locator")?; - let zip64 = usize::try_from(le_u64(bytes, locator + 8)?) - .map_err(|_| "ZIP64 record offset out of range".to_owned())?; - if bytes.get(zip64..zip64 + 4) != Some(&ZIP64_EOCD[..]) { - return Err("ZIP64 locator points at no ZIP64 record".to_owned()); - } - count = le_u64(bytes, zip64 + 32)?; - start = le_u64(bytes, zip64 + 48)?; - } - let mut at = usize::try_from(start).map_err(|_| "central directory offset out of range")?; - let mut names = Vec::new(); - for _ in 0..count { - if bytes.get(at..at + 4) != Some(&CENTRAL_HEADER[..]) { - return Err(format!("no central directory header at offset {at}")); - } - let name_len = usize::from(le_u16(bytes, at + 28)?); - let extra_len = usize::from(le_u16(bytes, at + 30)?); - let comment_len = usize::from(le_u16(bytes, at + 32)?); - let name = bytes - .get(at + 46..at + 46 + name_len) - .ok_or("truncated central directory header")?; - names.push(String::from_utf8_lossy(name).into_owned()); - at += 46 + name_len + extra_len + comment_len; - } - Ok(names) -} - -fn le_u16(bytes: &[u8], at: usize) -> std::result::Result { - bytes - .get(at..at + 2) - .map(|b| u16::from_le_bytes([b[0], b[1]])) - .ok_or_else(|| format!("truncated record at offset {at}")) -} - -fn le_u32(bytes: &[u8], at: usize) -> std::result::Result { - bytes - .get(at..at + 4) - .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .ok_or_else(|| format!("truncated record at offset {at}")) -} - -fn le_u64(bytes: &[u8], at: usize) -> std::result::Result { - bytes - .get(at..at + 8) - .map(|b| u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])) - .ok_or_else(|| format!("truncated record at offset {at}")) -} - -/// Reads one entry: `Ok(None)` when absent, `Err(detail)` when the -/// container cannot deliver it (a CRC mismatch included). -fn read_entry( - archive: &mut Archive<'_>, - name: &str, -) -> std::result::Result>, String> { - match archive.by_name(name) { - Ok(mut file) => { - let mut bytes = Vec::new(); - file.read_to_end(&mut bytes) - .map_err(|err| err.to_string())?; - Ok(Some(bytes)) - } - Err(zip::result::ZipError::FileNotFound) => Ok(None), - Err(err) => Err(err.to_string()), - } -} diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs index 7d69555..f6463c8 100644 --- a/crates/consolebook-server/tests/record_export.rs +++ b/crates/consolebook-server/tests/record_export.rs @@ -12,13 +12,14 @@ use axum::http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, COOKIE, SET_COOKIE}; use axum::http::{HeaderMap, Request, StatusCode}; use consolebook_server::capabilities::RoleBundle; use consolebook_server::draft_content::{self, DraftContent, NarrativeEntry, RatingEntry}; +use consolebook_server::export_verify::{self, Finding, PredecessorLink, UnitReport}; use consolebook_server::programs::{ self, AnchorDef, CompetencyDef, FormCompetencyDef, FormDef, NarrativeDef, PolicyDef, RecordType, ScaleDef, ScaleKind, VersionContent, }; use consolebook_server::record_export::{ self, ARCHIVE_FORMAT, ARCHIVE_MANIFEST_PATH, ArchiveManifest, ExportRefusal, FORMAT_VERSION, - Finding, PredecessorLink, Scope, UNIT_FORMAT, UnitManifest, + Scope, UNIT_FORMAT, UnitManifest, }; use consolebook_server::training_sessions::{self, Disposition, SessionInput}; use consolebook_server::{ @@ -605,7 +606,7 @@ async fn exports_carry_stored_bytes_verbatim_and_verify() { // Verification from the archive alone: the chain hash recomputes // from the carried predecessor hash, and the predecessor itself is // honestly reported as not in this export. - let report = record_export::verify_archive(&exported.bytes); + let report = export_verify::verify_archive(&exported.bytes); assert!(report.verified(), "{report:?}"); assert_eq!( report.installation_id.as_deref(), @@ -646,7 +647,7 @@ async fn exports_carry_stored_bytes_verbatim_and_verify() { assert_eq!(manifest.units[0].chain_hash, v1_chain); assert_eq!(manifest.units[0].predecessor_content_hash, None); assert_eq!(manifest.units[1].version_number, 2); - let report = record_export::verify_archive(&bytes); + let report = export_verify::verify_archive(&bytes); assert!(report.verified(), "{report:?}"); assert_eq!(report.units[0].predecessor, PredecessorLink::None); assert_eq!(report.units[1].predecessor, PredecessorLink::Linked); @@ -661,10 +662,10 @@ async fn exports_carry_stored_bytes_verbatim_and_verify() { }, ) .await; - assert!(record_export::verify_archive(&bytes).verified()); + assert!(export_verify::verify_archive(&bytes).verified()); assert_eq!(fx.audit_count(Some("enrollment")).await, 1); let bytes = export(&fx, fx.admin_id, Scope::Installation).await; - let report = record_export::verify_archive(&bytes); + let report = export_verify::verify_archive(&bytes); assert!(report.verified(), "{report:?}"); assert_eq!(report.scope, Some(Scope::Installation)); assert_eq!(report.units.len(), 2); @@ -698,7 +699,7 @@ async fn verification_names_every_finding() { let (v1_bytes, _, _) = fx.version_row(s.record_id, 1).await; // Not an archive at all. - let report = record_export::verify_archive(b"not a zip archive"); + let report = export_verify::verify_archive(b"not a zip archive"); assert!(!report.verified()); assert!(matches!(report.findings[0], Finding::NotAnArchive { .. })); @@ -715,7 +716,7 @@ async fn verification_names_every_finding() { ) }, )); - let report = record_export::verify_archive(&altered); + let report = export_verify::verify_archive(&altered); assert!(!report.verified()); assert!(report.findings.is_empty(), "{report:?}"); assert!(report.units[0].verified()); @@ -744,7 +745,7 @@ async fn verification_names_every_finding() { .position(|window| window == needle) .expect("record text in the archive"); flipped[at] ^= 0x20; - assert!(!record_export::verify_archive(&flipped).verified()); + assert!(!export_verify::verify_archive(&flipped).verified()); // The archive manifest claims a different content hash for version // 1: the bytes disagree, the unit manifest disagrees, and version @@ -755,7 +756,7 @@ async fn verification_names_every_finding() { manifest["units"][0]["content_hash"] = serde_json::Value::String(other_hash.clone()); })) })); - let report = record_export::verify_archive(&swapped); + let report = export_verify::verify_archive(&swapped); assert!(!report.verified()); assert!(report.findings.is_empty(), "{report:?}"); let findings = &report.units[0].findings; @@ -785,7 +786,7 @@ async fn verification_names_every_finding() { let replaced = repack(&with_entry(&listed, &format!("{v2}/record.json"), |_| { Some(v1_bytes.clone()) })); - let report = record_export::verify_archive(&replaced); + let report = export_verify::verify_archive(&replaced); let findings = &report.units[1].findings; assert!( findings.contains(&Finding::ContentHashMismatch), @@ -806,7 +807,7 @@ async fn verification_names_every_finding() { // Missing and unlisted entries. let missing = repack(&with_entry(&listed, &format!("{v1}/record.json"), |_| None)); - let report = record_export::verify_archive(&missing); + let report = export_verify::verify_archive(&missing); assert!( report.units[0].findings.contains(&Finding::MissingEntry { path: format!("{v1}/record.json") @@ -815,7 +816,7 @@ async fn verification_names_every_finding() { ); let mut extra = listed.clone(); extra.push(("README.txt".to_owned(), b"nothing to see here".to_vec())); - let report = record_export::verify_archive(&repack(&extra)); + let report = export_verify::verify_archive(&repack(&extra)); assert!(!report.verified()); assert!( report.findings.contains(&Finding::UnlistedEntry { @@ -823,7 +824,7 @@ async fn verification_names_every_finding() { }), "{report:?}" ); - assert!(report.units.iter().all(record_export::UnitReport::verified)); + assert!(report.units.iter().all(UnitReport::verified)); // A manifest that parses but is not canonical (pretty-printed) is a // finding: the format fixes the bytes, not only the members. @@ -835,7 +836,7 @@ async fn verification_names_every_finding() { Some(serde_json::to_vec_pretty(&value).expect("pretty")) }, )); - let report = record_export::verify_archive(&pretty); + let report = export_verify::verify_archive(&pretty); assert!( report.units[1] .findings @@ -851,7 +852,7 @@ async fn verification_names_every_finding() { manifest["format_version"] = serde_json::Value::from(2); })) })); - let report = record_export::verify_archive(&future); + let report = export_verify::verify_archive(&future); assert!(!report.verified()); assert!( report.findings.contains(&Finding::UnsupportedFormat { @@ -869,7 +870,7 @@ async fn verification_names_every_finding() { units.swap(0, 1); })) })); - let report = record_export::verify_archive(&reordered); + let report = export_verify::verify_archive(&reordered); assert!( report.findings.contains(&Finding::UnitsOutOfOrder), "{report:?}" @@ -879,7 +880,7 @@ async fn verification_names_every_finding() { manifest["units"][1]["path"] = serde_json::Value::String("records/elsewhere".into()); })) })); - let report = record_export::verify_archive(&renamed); + let report = export_verify::verify_archive(&renamed); assert!( report.findings.contains(&Finding::UnitPathUnexpected { path: "records/elsewhere".to_owned(), @@ -894,7 +895,7 @@ async fn verification_names_every_finding() { manifest["units"][1]["predecessor_content_hash"] = serde_json::Value::Null; })) })); - let report = record_export::verify_archive(&orphan); + let report = export_verify::verify_archive(&orphan); let findings = &report.units[1].findings; assert!(findings.contains(&Finding::LineageShape), "{findings:?}"); assert!( @@ -912,7 +913,7 @@ async fn verification_names_every_finding() { manifest["units"] = serde_json::Value::Array(Vec::new()); })) })); - let report = record_export::verify_archive(&hollow); + let report = export_verify::verify_archive(&hollow); assert!(!report.verified()); assert!(report.findings.contains(&Finding::NoUnits), "{report:?}"); let mislabeled = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { @@ -924,7 +925,7 @@ async fn verification_names_every_finding() { }); })) })); - let report = record_export::verify_archive(&mislabeled); + let report = export_verify::verify_archive(&mislabeled); assert!(!report.verified()); assert!( report.findings.contains(&Finding::ScopeCardinality { @@ -950,7 +951,7 @@ async fn verification_names_every_finding() { manifest["scope"]["record_id"] = serde_json::Value::from(9999); })) })); - let report = record_export::verify_archive(&foreign); + let report = export_verify::verify_archive(&foreign); assert!(!report.verified()); assert!( report @@ -974,7 +975,7 @@ async fn verification_names_every_finding() { .collect(); let handmade = handmade_zip(&plain); assert!( - record_export::verify_archive(&handmade).verified(), + export_verify::verify_archive(&handmade).verified(), "a hand-assembled container of the same entries verifies" ); let alternate = String::from_utf8(entry(&listed, &format!("{v2}/record.json")).to_vec()) @@ -988,7 +989,7 @@ async fn verification_names_every_finding() { } doubled.push((name, content)); } - let report = record_export::verify_archive(&handmade_zip(&doubled)); + let report = export_verify::verify_archive(&handmade_zip(&doubled)); assert!(!report.verified()); assert!( report.findings.contains(&Finding::DuplicateEntry { @@ -997,12 +998,12 @@ async fn verification_names_every_finding() { "{report:?}" ); assert!( - report.units.iter().all(record_export::UnitReport::verified), + report.units.iter().all(UnitReport::verified), "the reader saw the genuine copy: {report:?}" ); // The untouched original still verifies after all that. - assert!(record_export::verify_archive(&original).verified()); + assert!(export_verify::verify_archive(&original).verified()); } #[tokio::test] @@ -1030,7 +1031,7 @@ async fn scopes_follow_the_read_rules() { // outside the scope is refused. for actor in [s.jordan_id, s.casey_id, s.taylor_id] { let bytes = export(&fx, actor, record).await; - let report = record_export::verify_archive(&bytes); + let report = export_verify::verify_archive(&bytes); assert!(report.verified()); assert_eq!(report.units.len(), 2, "actor {actor} sees both versions"); export(&fx, actor, version).await; @@ -1180,7 +1181,7 @@ async fn export_api_delivers_the_documented_bytes() { )) && disposition.ends_with("Z.zip\""), "got: {disposition}" ); - let report = record_export::verify_archive(&bytes); + let report = export_verify::verify_archive(&bytes); assert!(report.verified(), "{report:?}"); assert_eq!(report.units.len(), 1); @@ -1191,7 +1192,7 @@ async fn export_api_delivers_the_documented_bytes() { ) .await; assert_eq!(status, StatusCode::OK); - assert_eq!(record_export::verify_archive(&bytes).units.len(), 2); + assert_eq!(export_verify::verify_archive(&bytes).units.len(), 2); let (status, _, bytes) = raw_get( fx.app(), &format!("/api/enrollments/{}/export", s.enrollment_id), @@ -1199,7 +1200,7 @@ async fn export_api_delivers_the_documented_bytes() { ) .await; assert_eq!(status, StatusCode::OK); - assert!(record_export::verify_archive(&bytes).verified()); + assert!(export_verify::verify_archive(&bytes).verified()); // The installation scope answers to export_records only. let (status, _, body) = raw_get(fx.app(), "/api/exports/records", &casey).await; @@ -1208,7 +1209,7 @@ async fn export_api_delivers_the_documented_bytes() { assert_eq!(body["error"], "capability_required"); let (status, _, bytes) = raw_get(fx.app(), "/api/exports/records", &admin).await; assert_eq!(status, StatusCode::OK); - let report = record_export::verify_archive(&bytes); + let report = export_verify::verify_archive(&bytes); assert!(report.verified()); assert_eq!(report.scope, Some(Scope::Installation)); let (status, _, body) = raw_get(fx.app(), "/api/exports/summary", &admin).await; diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md index 6356b98..f0cb7fb 100644 --- a/docs/formats/record-export.md +++ b/docs/formats/record-export.md @@ -2,10 +2,11 @@ Portable, verifiable exports of finalized evaluation versions (ADR 0014; `docs/records-integrity.md`; #44 decision 1). -`consolebook-server/src/record_export.rs` implements it; +`consolebook-server/src/record_export.rs` produces exports and +`consolebook-server/src/export_verify.rs` verifies them; `tests/record_export.rs` proves the round trip, determinism, and every -verification finding. This document -is normative: the implementation follows it, not the other way around. +verification finding. This document is normative: the implementation +follows it, not the other way around. An export never re-serializes a record. The stored canonical bytes of each version (ADR 0011, ADR 0013) travel byte for byte, beside a From b02e19e698d5e7030f7435e6ec7aaff89c8cf166 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 19:58:02 +0000 Subject: [PATCH 4/7] records(export): verify record bytes as typed envelopes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verifier parsed record.json as an arbitrary document and checked six identity members, so a canonical, correctly hashed document that was not a record — only those members, or a mistyped or missing presentation member — still verified. - record_envelope: the typed reading side of ADR 0011's shape (schema 1) and ADR 0013's (schema 2): every named member with its type, no unnamed member anywhere, every nullable member present, and daily_reports present exactly for schema 2. - export_verify reads record.json through it before the identity cross-check; a document that is not an envelope of a known schema is a typed `envelope_invalid` finding. - docs/formats/record-export.md names the check (unit check 4); ADR 0014's verification bullet points at the reader. - tests: a six-member counterfeit with matching hashes, a schema-2 envelope without daily_reports, an unnamed member, a mistyped member, an absent nullable member, an unknown schema — each invalid and nothing else; the same content shaped as schema 1 verifies; the genuine bytes read as the typed envelope they are. Refs #45, #44 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- .../consolebook-server/src/export_verify.rs | 59 ++-- crates/consolebook-server/src/lib.rs | 1 + .../consolebook-server/src/record_envelope.rs | 295 ++++++++++++++++++ .../consolebook-server/tests/record_export.rs | 155 +++++++++ docs/decisions/0014-record-export-format.md | 7 +- docs/formats/record-export.md | 12 +- 6 files changed, 499 insertions(+), 30 deletions(-) create mode 100644 crates/consolebook-server/src/record_envelope.rs diff --git a/crates/consolebook-server/src/export_verify.rs b/crates/consolebook-server/src/export_verify.rs index 163426b..7680750 100644 --- a/crates/consolebook-server/src/export_verify.rs +++ b/crates/consolebook-server/src/export_verify.rs @@ -20,6 +20,7 @@ use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; use crate::canonical; +use crate::record_envelope; use crate::record_export::{ ARCHIVE_FORMAT, ARCHIVE_MANIFEST_PATH, ArchiveManifest, FORMAT_VERSION, RECORD_FILE, Scope, UNIT_FORMAT, UNIT_MANIFEST_FILE, UnitEntry, UnitManifest, canonical_json, unit_path, @@ -99,6 +100,11 @@ pub enum Finding { EnvelopeDisagrees { member: &'static str, }, + /// The bytes are not an envelope of any known record schema: a + /// member missing, unnamed by the schema, or of the wrong type. + EnvelopeInvalid { + detail: String, + }, ChainHashMismatch, /// A first version with a predecessor, or a later one without. LineageShape, @@ -110,6 +116,9 @@ pub enum Finding { impl fmt::Display for Finding { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::EnvelopeInvalid { detail } => { + write!(f, "the record bytes are not a valid envelope: {detail}") + } Self::NotAnArchive { detail } => write!(f, "not a readable ZIP archive: {detail}"), Self::ArchiveManifestMissing => f.write_str("the archive manifest is missing"), Self::ArchiveManifestUnreadable { detail } => { @@ -424,41 +433,43 @@ fn verify_unit( findings.push(Finding::ContentHashMismatch); } match serde_json::from_slice::(&bytes) { + Ok(document) => match canonical::canonical_bytes(&document) { + Ok(again) if again == bytes => {} + Ok(_) => findings.push(Finding::NotCanonical { + detail: "re-serialization differs from the stored bytes".to_owned(), + }), + Err(err) => findings.push(Finding::NotCanonical { + detail: err.to_string(), + }), + }, + Err(err) => findings.push(Finding::NotCanonical { + detail: format!("not JSON: {err}"), + }), + } + // The bytes must be an envelope of a known record schema — + // every member the schema names, typed, and no other — before + // the identity they carry is compared with the manifest. + match record_envelope::parse(&bytes) { Ok(envelope) => { - match canonical::canonical_bytes(&envelope) { - Ok(again) if again == bytes => {} - Ok(_) => findings.push(Finding::NotCanonical { - detail: "re-serialization differs from the stored bytes".to_owned(), - }), - Err(err) => findings.push(Finding::NotCanonical { - detail: err.to_string(), - }), - } - let predecessor = entry - .predecessor_content_hash - .as_ref() - .map_or(Value::Null, |hash| Value::String(hash.clone())); let disagreements: [(&'static str, bool); 6] = [ - ("record.id", envelope["record"]["id"] != entry.record_id), + ("record.id", envelope.record.id != entry.record_id), ( "record.version_number", - envelope["record"]["version_number"] != entry.version_number, + envelope.record.version_number != entry.version_number, ), ( "record.record_schema", - envelope["record"]["record_schema"] != entry.record_schema, + envelope.record.record_schema != entry.record_schema, ), ( "record.predecessor_content_hash", - envelope["record"]["predecessor_content_hash"] != predecessor, - ), - ( - "instance", - envelope["instance"] != manifest.installation_id.as_str(), + envelope.record.predecessor_content_hash + != entry.predecessor_content_hash, ), + ("instance", envelope.instance != manifest.installation_id), ( "canonicalization", - envelope["canonicalization"] != canonical::CANONICALIZATION, + envelope.canonicalization != canonical::CANONICALIZATION, ), ]; for (member, disagrees) in disagreements { @@ -467,8 +478,8 @@ fn verify_unit( } } } - Err(err) => findings.push(Finding::NotCanonical { - detail: format!("not JSON: {err}"), + Err(err) => findings.push(Finding::EnvelopeInvalid { + detail: err.to_string(), }), } match canonical::chain_hash_hex(entry.predecessor_content_hash.as_deref(), &bytes) { diff --git a/crates/consolebook-server/src/lib.rs b/crates/consolebook-server/src/lib.rs index 881a3e8..91983bd 100644 --- a/crates/consolebook-server/src/lib.rs +++ b/crates/consolebook-server/src/lib.rs @@ -27,6 +27,7 @@ pub mod notices; pub mod program_export; pub mod programs; pub mod programs_http; +pub mod record_envelope; pub mod record_export; pub mod restore; pub mod scheduler; diff --git a/crates/consolebook-server/src/record_envelope.rs b/crates/consolebook-server/src/record_envelope.rs new file mode 100644 index 0000000..a723e43 --- /dev/null +++ b/crates/consolebook-server/src/record_envelope.rs @@ -0,0 +1,295 @@ +//! The typed shape of a finalized record envelope, as readers see it +//! (ADR 0011 for record schema 1, ADR 0013 for schema 2). +//! +//! `finalization::envelope` is the producing side and the one owner of +//! what goes into a record. This is the reading side: the same member +//! set, typed, with every object refusing members the schema does not +//! name and every nullable member required to be present, so "these +//! bytes are a schema-1 or schema-2 envelope" is a typed contract +//! rather than a spot check of a few members. Export verification reads +//! through it; packets and rendering (Milestone 5) will too. A schema +//! bump extends this shape and never reinterprets stored bytes. + +use std::fmt; + +use serde::{Deserialize, Deserializer}; + +use crate::canonical; + +/// A user as the envelope presents them: identity plus the names shown +/// at finalization. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct User { + pub id: i64, + pub username: String, + pub display_name: String, +} + +/// Attachments exist in no schema yet: the array is always empty, and +/// an element of any shape is not a schema-1 or schema-2 record. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Attachment {} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttributionEvent { + pub kind: String, + pub actor: User, + #[serde(deserialize_with = "nullable")] + pub to: Option, + pub recorded_at: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Content { + pub narratives: Vec, + pub ratings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Narrative { + pub prompt: String, + pub required: bool, + #[serde(deserialize_with = "nullable")] + pub text: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rating { + pub competency: Competency, + pub scale: Scale, + #[serde(deserialize_with = "nullable")] + pub value: Option, + pub not_observed: bool, + pub modifiers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Competency { + pub category: String, + pub name: String, + pub description: String, + pub tasks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scale { + pub name: String, + pub kind: String, + #[serde(deserialize_with = "nullable")] + pub min_value: Option, + #[serde(deserialize_with = "nullable")] + pub max_value: Option, + pub anchors: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Anchor { + pub value: i64, + pub label: String, + pub definition: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Modifier { + pub code: String, + pub label: String, + pub description: String, +} + +/// One pinned daily-report version a weekly summary covered (schema 2). +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DailyReport { + pub content_hash: String, + pub record_id: i64, + pub version_number: i64, +} + +/// The `daily_reports` member: absent in schema 1, an array in schema +/// 2. Three states are told apart — absent, `null` (never valid), and +/// present — because schema 1 requires the member to be missing. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum DailyReports { + #[default] + Absent, + Present(Vec), +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Finalization { + pub finalized_at: i64, + pub finalized_by: User, + pub policy: Policy, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Policy { + pub review_approved: bool, + pub required_narratives: bool, + pub ratings_complete: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Form { + pub name: String, + pub instructions: String, + pub record_type: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Program { + pub name: String, + pub version_number: i64, + pub label: String, +} + +/// The record's own statement of identity and lineage. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordIdentity { + pub id: i64, + pub version_number: i64, + pub record_schema: i64, + #[serde(deserialize_with = "nullable")] + pub predecessor_content_hash: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReviewDecision { + pub reviewer: User, + pub decision: String, + pub comment: String, + pub decided_at: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Session { + pub business_date: String, + pub timezone: String, + pub local_start: String, + #[serde(deserialize_with = "nullable")] + pub local_end: Option, + pub utc_start: i64, + #[serde(deserialize_with = "nullable")] + pub utc_end: Option, + #[serde(deserialize_with = "nullable")] + pub disposition: Option, + #[serde(deserialize_with = "nullable")] + pub phase: Option, + pub trainers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Phase { + pub name: String, + pub presentation_number: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Trainee { + pub id: i64, + pub username: String, + pub display_name: String, + pub employee_id: String, + pub title: String, +} + +/// The envelope's top level: every member ADR 0011 names, plus +/// `daily_reports` for schema 2 (ADR 0013), and nothing else. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Envelope { + pub attachments: Vec, + pub attribution: Vec, + pub canonicalization: String, + pub content: Content, + #[serde(default, deserialize_with = "daily_reports")] + pub daily_reports: DailyReports, + pub finalization: Finalization, + pub form: Form, + pub instance: String, + pub program: Program, + pub record: RecordIdentity, + pub review: Vec, + pub sessions: Vec, + pub trainee: Trainee, +} + +/// A member that may be `null` but must be present: the producer always +/// writes it, so a document without it is not an envelope. +fn nullable<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: Deserialize<'de>, + D: Deserializer<'de>, +{ + Option::::deserialize(deserializer) +} + +fn daily_reports<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Vec::::deserialize(deserializer).map(DailyReports::Present) +} + +/// Why bytes are not an envelope of any known schema. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EnvelopeError { + /// Not the member set and types of any schema this build knows. + Malformed(String), + /// A `record.record_schema` this build does not know. + UnsupportedSchema(i64), + /// `daily_reports` is present exactly in schema 2. + SchemaShape(i64), +} + +impl fmt::Display for EnvelopeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Malformed(detail) => write!(f, "not a record envelope: {detail}"), + Self::UnsupportedSchema(schema) => { + write!(f, "unsupported record schema {schema}") + } + Self::SchemaShape(schema) => write!( + f, + "record schema {schema} and the daily_reports member disagree" + ), + } + } +} + +impl std::error::Error for EnvelopeError {} + +/// Reads bytes as an envelope of the schema they declare, refusing any +/// document that is not exactly a schema-1 or schema-2 record. +pub fn parse(bytes: &[u8]) -> Result { + let envelope: Envelope = + serde_json::from_slice(bytes).map_err(|err| EnvelopeError::Malformed(err.to_string()))?; + let schema = envelope.record.record_schema; + if !(1..=canonical::RECORD_SCHEMA).contains(&schema) { + return Err(EnvelopeError::UnsupportedSchema(schema)); + } + let has_daily_reports = matches!(envelope.daily_reports, DailyReports::Present(_)); + if has_daily_reports != (schema == 2) { + return Err(EnvelopeError::SchemaShape(schema)); + } + Ok(envelope) +} diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs index f6463c8..a21660c 100644 --- a/crates/consolebook-server/tests/record_export.rs +++ b/crates/consolebook-server/tests/record_export.rs @@ -488,6 +488,41 @@ fn handmade_zip(entries: &[(&str, &[u8])]) -> Vec { out } +/// A single-unit archive whose record bytes are replaced and whose +/// manifests are rehashed and relabeled to match: the honest way to +/// forge a unit, leaving only the envelope's own shape to object. +fn rehashed_unit( + listed: &[(String, Vec)], + unit: &str, + bytes: &[u8], + predecessor: Option<&str>, + record_schema: i64, +) -> Vec { + let content_hash = canonical::content_hash_hex(bytes); + let chain_hash = canonical::chain_hash_hex(predecessor, bytes).expect("chain"); + let relabel = |manifest: &mut serde_json::Value| { + manifest["content_hash"] = serde_json::Value::String(content_hash.clone()); + manifest["chain_hash"] = serde_json::Value::String(chain_hash.clone()); + manifest["record_schema"] = serde_json::Value::from(record_schema); + }; + let entries: Vec<(String, Vec)> = listed + .iter() + .map(|(name, content)| { + let content = if *name == format!("{unit}/record.json") { + bytes.to_vec() + } else if name == ARCHIVE_MANIFEST_PATH { + edit_json(content, |manifest| relabel(&mut manifest["units"][0])) + } else if *name == format!("{unit}/manifest.json") { + edit_json(content, relabel) + } else { + content.clone() + }; + (name.clone(), content) + }) + .collect(); + repack(&entries) +} + fn edit_json(bytes: &[u8], edit: impl FnOnce(&mut serde_json::Value)) -> Vec { let mut value: serde_json::Value = serde_json::from_slice(bytes).expect("json"); edit(&mut value); @@ -1006,6 +1041,126 @@ async fn verification_names_every_finding() { assert!(export_verify::verify_archive(&original).verified()); } +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn verification_reads_envelopes_as_typed_records() { + let fx = Fixture::new().await; + let s = seed(&fx, "typed").await; + let installation_id = storage::installation_id(&fx.pool).await.expect("id"); + let (_, v1_content, _) = fx.version_row(s.record_id, 1).await; + let original = export( + &fx, + s.casey_id, + Scope::Version { + record_id: s.record_id, + version_number: 2, + }, + ) + .await; + let listed = entries(&original); + let unit = format!("records/{}/v2", s.record_id); + let genuine: serde_json::Value = + serde_json::from_slice(entry(&listed, &format!("{unit}/record.json"))).expect("json"); + let invalid = |bytes: &[u8], record_schema: i64| { + let report = export_verify::verify_archive(&rehashed_unit( + &listed, + &unit, + bytes, + Some(&v1_content), + record_schema, + )); + assert!(!report.verified(), "{report:?}"); + let findings = &report.units[0].findings; + assert!( + findings + .iter() + .any(|f| matches!(f, Finding::EnvelopeInvalid { .. })), + "{findings:?}" + ); + assert!( + !findings.contains(&Finding::ContentHashMismatch) + && !findings.contains(&Finding::ChainHashMismatch) + && !findings + .iter() + .any(|f| matches!(f, Finding::NotCanonical { .. })), + "the forgery is otherwise self-consistent: {findings:?}" + ); + }; + + // A canonical, correctly hashed document carrying only the members + // the identity cross-check reads is not a record. + let counterfeit = canonical::canonical_bytes(&serde_json::json!({ + "canonicalization": canonical::CANONICALIZATION, + "instance": installation_id, + "record": { + "id": s.record_id, + "version_number": 2, + "record_schema": 2, + "predecessor_content_hash": v1_content, + }, + })) + .expect("canonical"); + invalid(&counterfeit, 2); + + // Nor is a schema-2 envelope missing its daily_reports member, one + // carrying a member no schema names, one with a mistyped member, one + // whose nullable member is absent instead of null, or one declaring + // a schema this build does not know. + let mut without = genuine.clone(); + without + .as_object_mut() + .expect("object") + .remove("daily_reports"); + invalid(&canonical::canonical_bytes(&without).expect("canonical"), 2); + let mut extra = genuine.clone(); + extra["annotations"] = serde_json::json!([]); + invalid(&canonical::canonical_bytes(&extra).expect("canonical"), 2); + let mut retyped = genuine.clone(); + retyped["finalization"]["finalized_at"] = serde_json::json!("yesterday"); + invalid(&canonical::canonical_bytes(&retyped).expect("canonical"), 2); + let mut absent = genuine.clone(); + absent["content"]["narratives"][0] + .as_object_mut() + .expect("object") + .remove("text"); + invalid(&canonical::canonical_bytes(&absent).expect("canonical"), 2); + let mut unknown_schema = genuine.clone(); + unknown_schema["record"]["record_schema"] = serde_json::json!(7); + invalid( + &canonical::canonical_bytes(&unknown_schema).expect("canonical"), + 7, + ); + + // The same content shaped as schema 1 — no daily_reports, schema 1 + // declared throughout — is a valid record of its own schema. + let mut schema1 = without; + schema1["record"]["record_schema"] = serde_json::json!(1); + let report = export_verify::verify_archive(&rehashed_unit( + &listed, + &unit, + &canonical::canonical_bytes(&schema1).expect("canonical"), + Some(&v1_content), + 1, + )); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.units[0].record_schema, 1); + + // And the genuine bytes read as the typed envelope they are. + let envelope = + consolebook_server::record_envelope::parse(entry(&listed, &format!("{unit}/record.json"))) + .expect("typed envelope"); + assert_eq!(envelope.record.version_number, 2); + assert_eq!(envelope.trainee.display_name, "Taylor Trainee"); + assert_eq!( + envelope.content.narratives[0].text.as_deref(), + Some("Corrected the invented rating with context.") + ); + assert!(matches!( + envelope.daily_reports, + consolebook_server::record_envelope::DailyReports::Present(ref links) if links.is_empty() + )); +} + #[tokio::test] #[allow(clippy::too_many_lines)] async fn scopes_follow_the_read_rules() { diff --git a/docs/decisions/0014-record-export-format.md b/docs/decisions/0014-record-export-format.md index ccdbe62..a754d4d 100644 --- a/docs/decisions/0014-record-export-format.md +++ b/docs/decisions/0014-record-export-format.md @@ -51,8 +51,11 @@ may export, and what verification claims (#45; Milestone 5 slice 1). ### Verification from the export alone - The verifier recomputes the content hash over the bytes, proves the - bytes are canonical by re-serialization, cross-checks the envelope's - own `record` and `instance` members against the manifests, + bytes are canonical by re-serialization, reads them as a typed + envelope of their declared record schema (`record_envelope`, the + reading side of ADR 0011's shape: every named member, typed, and no + other), cross-checks the envelope's own `record` and `instance` + members against the manifests, recomputes the chain hash from the carried predecessor hash, links predecessors present in the same archive, checks the listed units against the declared scope as far as the archive allows, and walks diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md index f0cb7fb..3d59ed2 100644 --- a/docs/formats/record-export.md +++ b/docs/formats/record-export.md @@ -207,17 +207,21 @@ Unit checks, for every listed unit: 3. `record.json` parses as JSON and re-serializing it under the canonical subset reproduces the identical bytes — the bytes are canonical, so the hash is over the specified representation; -4. the envelope agrees with the manifest: `record.id`, +4. `record.json` is an envelope of a known record schema: every member + ADR 0011 (schema 1) or ADR 0013 (schema 2) names, with its type, + every nullable member present, no member the schema does not name, + and `daily_reports` present exactly for schema 2; +5. the envelope agrees with the manifest: `record.id`, `record.version_number`, `record.record_schema`, `record.predecessor_content_hash`, `instance`, and `canonicalization` (`jcs-v1`); -5. `chain_hash` equals +6. `chain_hash` equals `SHA-256("consolebook-version-v1" || 0x00 || predecessor || bytes)` with `predecessor` the raw 32 bytes of `predecessor_content_hash`, or 32 zero bytes when it is `null`; -6. `predecessor_content_hash` is `null` exactly when `version_number` +7. `predecessor_content_hash` is `null` exactly when `version_number` is 1; and -7. when the archive also lists (`record_id`, `version_number - 1`), +8. when the archive also lists (`record_id`, `version_number - 1`), that unit's `content_hash` equals this unit's `predecessor_content_hash` — reported as *linked*; a predecessor the archive does not carry is reported as *not in export*, which is not From 6fe27df352bfafbc55c5cdb8f663a14c9eab3c31 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 20:05:41 +0000 Subject: [PATCH 5/7] records(export): refuse attachment elements and non-positive identity Two gaps in the new validation surface from review: - The typed envelope reader accepted "attachments":[{}] because the element type was an empty struct; both known schemas require the array empty. The element type is now uninhabited, so no value of any shape deserializes into it. - The lineage rule (predecessor null exactly for version 1) is satisfied trivially by version 0 or a negative number with a predecessor, and the predecessor lookup then reports "not in export" without objecting. Identity is positive by the format; a zero or negative record_id or version_number is the typed finding identity_out_of_range. docs/formats/record-export.md names both (unit checks 4 and 7); tests cover an attachment element and a renumbered version 0 unit whose only objection is the range check. Refs #45, #44 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- .../consolebook-server/src/export_verify.rs | 20 +++++++ .../consolebook-server/src/record_envelope.rs | 21 ++++++-- .../consolebook-server/tests/record_export.rs | 53 +++++++++++++++++++ docs/formats/record-export.md | 8 +-- 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/crates/consolebook-server/src/export_verify.rs b/crates/consolebook-server/src/export_verify.rs index 7680750..d525999 100644 --- a/crates/consolebook-server/src/export_verify.rs +++ b/crates/consolebook-server/src/export_verify.rs @@ -108,6 +108,10 @@ pub enum Finding { ChainHashMismatch, /// A first version with a predecessor, or a later one without. LineageShape, + /// `record_id` and `version_number` are positive integers. + IdentityOutOfRange { + member: &'static str, + }, /// The predecessor is in the archive and its content hash is not /// what this unit's chain was computed over. PredecessorMismatch, @@ -116,6 +120,9 @@ pub enum Finding { impl fmt::Display for Finding { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::IdentityOutOfRange { member } => { + write!(f, "{member} is not a positive integer") + } Self::EnvelopeInvalid { detail } => { write!(f, "the record bytes are not a valid envelope: {detail}") } @@ -496,6 +503,19 @@ fn verify_unit( }), } + // Identity is positive by the format: the database assigns record + // ids from 1 and version numbers start at 1, so a zero or negative + // number is not an identity the lineage rule below can reason about. + if entry.record_id < 1 { + findings.push(Finding::IdentityOutOfRange { + member: "record_id", + }); + } + if entry.version_number < 1 { + findings.push(Finding::IdentityOutOfRange { + member: "version_number", + }); + } if (entry.version_number == 1) != entry.predecessor_content_hash.is_none() { findings.push(Finding::LineageShape); } diff --git a/crates/consolebook-server/src/record_envelope.rs b/crates/consolebook-server/src/record_envelope.rs index a723e43..f736c30 100644 --- a/crates/consolebook-server/src/record_envelope.rs +++ b/crates/consolebook-server/src/record_envelope.rs @@ -26,11 +26,22 @@ pub struct User { pub display_name: String, } -/// Attachments exist in no schema yet: the array is always empty, and -/// an element of any shape is not a schema-1 or schema-2 record. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Attachment {} +/// Attachments exist in no schema yet: the array is always empty. The +/// element type is uninhabited, so no value of any shape deserializes +/// into it and a non-empty array is not a schema-1 or schema-2 record. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Attachment {} + +impl<'de> Deserialize<'de> for Attachment { + fn deserialize(_: D) -> Result + where + D: Deserializer<'de>, + { + Err(::custom( + "attachments exist in no known record schema", + )) + } +} #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs index a21660c..2611879 100644 --- a/crates/consolebook-server/tests/record_export.rs +++ b/crates/consolebook-server/tests/record_export.rs @@ -1130,6 +1130,59 @@ async fn verification_reads_envelopes_as_typed_records() { &canonical::canonical_bytes(&unknown_schema).expect("canonical"), 7, ); + // Attachments exist in no known schema: an element of any shape, + // even an empty object, is not a record. + let mut attached = genuine.clone(); + attached["attachments"] = serde_json::json!([{}]); + invalid( + &canonical::canonical_bytes(&attached).expect("canonical"), + 2, + ); + + // Identity is positive: a version 0 with a predecessor satisfies the + // lineage rule's two sides trivially, so the range check has to be + // its own finding — and it is the only objection. + let v0 = format!("records/{}/v0", s.record_id); + let mut zeroed = genuine.clone(); + zeroed["record"]["version_number"] = serde_json::json!(0); + let zeroed_bytes = canonical::canonical_bytes(&zeroed).expect("canonical"); + let zeroed_content = canonical::content_hash_hex(&zeroed_bytes); + let zeroed_chain = canonical::chain_hash_hex(Some(&v1_content), &zeroed_bytes).expect("chain"); + let relabel_zero = |manifest: &mut serde_json::Value| { + manifest["version_number"] = serde_json::json!(0); + manifest["content_hash"] = serde_json::Value::String(zeroed_content.clone()); + manifest["chain_hash"] = serde_json::Value::String(zeroed_chain.clone()); + }; + let renumbered: Vec<(String, Vec)> = listed + .iter() + .map(|(name, content)| { + if name == ARCHIVE_MANIFEST_PATH { + let edited = edit_json(content, |manifest| { + manifest["scope"]["version_number"] = serde_json::json!(0); + relabel_zero(&mut manifest["units"][0]); + manifest["units"][0]["path"] = serde_json::Value::String(v0.clone()); + }); + (name.clone(), edited) + } else if name.ends_with("/record.json") { + (format!("{v0}/record.json"), zeroed_bytes.clone()) + } else { + ( + format!("{v0}/manifest.json"), + edit_json(content, relabel_zero), + ) + } + }) + .collect(); + let report = export_verify::verify_archive(&repack(&renumbered)); + assert!(!report.verified()); + assert_eq!( + report.units[0].findings, + vec![Finding::IdentityOutOfRange { + member: "version_number" + }], + "{report:?}" + ); + assert!(report.findings.is_empty(), "{report:?}"); // The same content shaped as schema 1 — no daily_reports, schema 1 // declared throughout — is a valid record of its own schema. diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md index 3d59ed2..96fc3ff 100644 --- a/docs/formats/record-export.md +++ b/docs/formats/record-export.md @@ -96,7 +96,7 @@ identity and lineage on their own (ADR 0011). | `format` | string | Always `consolebook-record-unit` | | `format_version` | integer | `1`; bumped by any change to either manifest's shape | | `installation_id` | string | The exporting installation's identity (`instance.installation_id`) | -| `record_id` | integer | The record's instance-local identity | +| `record_id` | integer | The record's instance-local identity (positive) | | `version_number` | integer | This version's number within the record (from 1) | | `record_schema` | integer | The stored envelope schema of `record.json` | | `content_hash` | string | The stored SHA-256 of `record.json`, lowercase hex | @@ -210,7 +210,8 @@ Unit checks, for every listed unit: 4. `record.json` is an envelope of a known record schema: every member ADR 0011 (schema 1) or ADR 0013 (schema 2) names, with its type, every nullable member present, no member the schema does not name, - and `daily_reports` present exactly for schema 2; + `daily_reports` present exactly for schema 2, and `attachments` + empty (no known schema carries attachments); 5. the envelope agrees with the manifest: `record.id`, `record.version_number`, `record.record_schema`, `record.predecessor_content_hash`, `instance`, and @@ -219,7 +220,8 @@ Unit checks, for every listed unit: `SHA-256("consolebook-version-v1" || 0x00 || predecessor || bytes)` with `predecessor` the raw 32 bytes of `predecessor_content_hash`, or 32 zero bytes when it is `null`; -7. `predecessor_content_hash` is `null` exactly when `version_number` +7. `record_id` and `version_number` are positive integers, and + `predecessor_content_hash` is `null` exactly when `version_number` is 1; and 8. when the archive also lists (`record_id`, `version_number - 1`), that unit's `content_hash` equals this unit's From f1c7aa3e8190fb5b4a1e5b005713970ad923b86c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 20:14:20 +0000 Subject: [PATCH 6/7] records(export): require lowercase hashes; release bytes as written - The chain recomputation decodes hex case-insensitively, so an uppercase predecessor hash carried consistently through both manifests and the envelope verified for a lone successor and would then fail to link beside its lowercase predecessor. Hash members are 64 lowercase hex characters by the format; a deviation is the typed finding hash_not_canonical (docs: unit check 2). - build_archive consumes its rows so each version's bytes are released once written: the assembled archive is the one copy held to the end, bounding the peak near one copy of the corpus instead of two. Streaming the response itself is #47; ADR 0014's cost note now says what an installation export costs until then. Refs #45, #44, #47 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- .../consolebook-server/src/export_verify.rs | 33 +++++++++++++++++++ .../consolebook-server/src/record_export.rs | 13 +++++--- .../consolebook-server/tests/record_export.rs | 27 +++++++++++++++ docs/decisions/0014-record-export-format.md | 6 ++-- docs/formats/record-export.md | 4 ++- 5 files changed, 75 insertions(+), 8 deletions(-) diff --git a/crates/consolebook-server/src/export_verify.rs b/crates/consolebook-server/src/export_verify.rs index d525999..1004e28 100644 --- a/crates/consolebook-server/src/export_verify.rs +++ b/crates/consolebook-server/src/export_verify.rs @@ -106,6 +106,10 @@ pub enum Finding { detail: String, }, ChainHashMismatch, + /// A hash member is not 64 lowercase hex characters. + HashNotCanonical { + member: &'static str, + }, /// A first version with a predecessor, or a later one without. LineageShape, /// `record_id` and `version_number` are positive integers. @@ -120,6 +124,9 @@ pub enum Finding { impl fmt::Display for Finding { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::HashNotCanonical { member } => { + write!(f, "{member} is not 64 lowercase hex characters") + } Self::IdentityOutOfRange { member } => { write!(f, "{member} is not a positive integer") } @@ -503,6 +510,25 @@ fn verify_unit( }), } + // Hashes are 64 lowercase hex characters by the format. The chain + // recomputation decodes hex case-insensitively, so without this an + // uppercase predecessor hash would pass for a lone successor and + // then fail to link once its lowercase predecessor joined it. + let hash_members: [(&'static str, Option<&str>); 3] = [ + ("content_hash", Some(entry.content_hash.as_str())), + ("chain_hash", Some(entry.chain_hash.as_str())), + ( + "predecessor_content_hash", + entry.predecessor_content_hash.as_deref(), + ), + ]; + for (member, value) in hash_members { + if let Some(hex) = value + && !is_lowercase_hex_hash(hex) + { + findings.push(Finding::HashNotCanonical { member }); + } + } // Identity is positive by the format: the database assigns record // ids from 1 and version numbers start at 1, so a zero or negative // number is not an identity the lineage rule below can reason about. @@ -681,6 +707,13 @@ fn le_u64(bytes: &[u8], at: usize) -> std::result::Result { .ok_or_else(|| format!("truncated record at offset {at}")) } +fn is_lowercase_hex_hash(hex: &str) -> bool { + hex.len() == 64 + && hex + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + /// Reads one entry: `Ok(None)` when absent, `Err(detail)` when the /// container cannot deliver it (a CRC mismatch included). fn read_entry( diff --git a/crates/consolebook-server/src/record_export.rs b/crates/consolebook-server/src/record_export.rs index c340e16..4171eed 100644 --- a/crates/consolebook-server/src/record_export.rs +++ b/crates/consolebook-server/src/record_export.rs @@ -183,7 +183,8 @@ pub async fn export_at( })); } let installation_id = storage::installation_id(pool).await?; - let bytes = build_archive(&installation_id, exported_at, scope, &rows)?; + let unit_count = rows.len(); + let bytes = build_archive(&installation_id, exported_at, scope, rows)?; // The export is audited once it exists: actor and subject, never // content (docs/records-integrity.md). match audited.subject { @@ -203,7 +204,7 @@ pub async fn export_at( file_name: file_name(scope, exported_at)?, bytes, exported_at, - unit_count: rows.len(), + unit_count, })) } @@ -378,12 +379,14 @@ async fn collect(pool: &SqlitePool, scope: Scope) -> Result> { /// Writes the container exactly as docs/formats/record-export.md lays /// it out: manifest first, then units in order, stored entries, the -/// export instant as every entry's modification time, `0644`. +/// export instant as every entry's modification time, `0644`. Rows are +/// consumed so each version's bytes are released once written; the +/// archive is the one copy held to the end (#47 tracks streaming it). fn build_archive( installation_id: &str, exported_at: i64, scope: Scope, - rows: &[VersionRow], + rows: Vec, ) -> Result> { let units: Vec = rows .iter() @@ -416,7 +419,7 @@ fn build_archive( &canonical_json(&manifest)?, options, )?; - for (row, entry) in rows.iter().zip(&manifest.units) { + for (row, entry) in rows.into_iter().zip(&manifest.units) { add_entry( &mut writer, &format!("{}/{RECORD_FILE}", entry.path), diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs index 2611879..1f5e4c0 100644 --- a/crates/consolebook-server/tests/record_export.rs +++ b/crates/consolebook-server/tests/record_export.rs @@ -504,6 +504,10 @@ fn rehashed_unit( manifest["content_hash"] = serde_json::Value::String(content_hash.clone()); manifest["chain_hash"] = serde_json::Value::String(chain_hash.clone()); manifest["record_schema"] = serde_json::Value::from(record_schema); + manifest["predecessor_content_hash"] = predecessor + .map_or(serde_json::Value::Null, |hash| { + serde_json::Value::String(hash.to_owned()) + }); }; let entries: Vec<(String, Vec)> = listed .iter() @@ -1130,6 +1134,29 @@ async fn verification_reads_envelopes_as_typed_records() { &canonical::canonical_bytes(&unknown_schema).expect("canonical"), 7, ); + // Hashes are lowercase hex by the format. An uppercase predecessor + // hash, consistent across both manifests and the envelope, decodes + // to the same bytes, so the chain recomputes — and nothing else + // objects for a lone successor whose predecessor is not present. + let shouted = v1_content.to_uppercase(); + let mut loud = genuine.clone(); + loud["record"]["predecessor_content_hash"] = serde_json::Value::String(shouted.clone()); + let report = export_verify::verify_archive(&rehashed_unit( + &listed, + &unit, + &canonical::canonical_bytes(&loud).expect("canonical"), + Some(&shouted), + 2, + )); + assert!(!report.verified()); + assert_eq!( + report.units[0].findings, + vec![Finding::HashNotCanonical { + member: "predecessor_content_hash" + }], + "{report:?}" + ); + // Attachments exist in no known schema: an element of any shape, // even an empty object, is not a record. let mut attached = genuine.clone(); diff --git a/docs/decisions/0014-record-export-format.md b/docs/decisions/0014-record-export-format.md index a754d4d..f3aa8ce 100644 --- a/docs/decisions/0014-record-export-format.md +++ b/docs/decisions/0014-record-export-format.md @@ -121,8 +121,10 @@ may export, and what verification claims (#45; Milestone 5 slice 1). exported successor without its predecessor verifies its own chain hash but cannot prove the predecessor's bytes — reported as *not in export*, never inferred; and -- streaming exports means nothing on disk to dispose of, and nothing - to resume: a very large installation export is a single response. +- on-demand exports mean nothing on disk to dispose of and nothing to + resume: an installation export is one response, assembled in memory + while it is produced, so a very large history costs memory in + proportion until exports stream (tracked separately). ## Rejected alternatives diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md index 96fc3ff..fb36648 100644 --- a/docs/formats/record-export.md +++ b/docs/formats/record-export.md @@ -203,7 +203,9 @@ Unit checks, for every listed unit: `format_version`, and agrees with the archive entry and the archive manifest on every shared member (`installation_id`, `exported_at`, identity, schema, hashes); -2. `content_hash` equals SHA-256 over the bytes of `record.json`; +2. `content_hash`, `chain_hash`, and a non-null + `predecessor_content_hash` are each 64 lowercase hex characters, and + `content_hash` equals SHA-256 over the bytes of `record.json`; 3. `record.json` parses as JSON and re-serializing it under the canonical subset reproduces the identical bytes — the bytes are canonical, so the hash is over the specified representation; From dd6640650f323dda331ccb5edeb7db963a99e103 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 20:19:59 +0000 Subject: [PATCH 7/7] docs(formats): say what export verification does not check Entry order, modification times, and permissions are production rules that make a fresh export deterministic; they carry no record content, and an intact archive stays verified after a tool repacks it. The format document now states this so the verifier's scope is explicit. Refs #45, #44 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DNP3dkn9c84v2YoNwx7KTg --- docs/formats/record-export.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md index fb36648..7ba7b47 100644 --- a/docs/formats/record-export.md +++ b/docs/formats/record-export.md @@ -236,6 +236,15 @@ Unit checks, for every listed unit: prints one line per unit and every finding, and exits non-zero unless the verdict is `verified`. It opens no data directory. +Verification deliberately does not check the container's entry order, +modification times, or permissions. Those are production rules: they +make a fresh export deterministic, and they carry no record content. +An archive whose entries and manifests are intact is still a verified +export after a tool has repacked it, which is what an operator asking +"are these records intact?" needs to hear. Proving an archive is +byte-identical to a fresh export is a byte comparison, not a +verification finding. + ## What the archive does not carry - **Drafts.** An unfinalized record is not a record; scopes contain