From 716f417e66ff3b4d601b27ad8fed67d129813d19 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 22:28:22 +0000 Subject: [PATCH 1/9] feat(planner): D-MBX-A6 persistence-sink ordering core (post-write kanbanstep) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The POST-write half of the fire-and-forget flow (pre-write half = owner_adapter, #877). Runs on the independent persistence path — the thinker already cast and moved on. persist_then_step(owner, write, paired, bytes): durable write FIRST, then — and only on a successful write — apply the cast's PAIRED move via the owner's checked try_advance_phase. Ordering invariant proven by 5 lance-free falsifiable probes: - no successful write => no step (owner untouched); - successful write => the PAIRED move applied exactly once; - applies paired.to NOT next_phases().first() — from Evaluation the generic successor is Commit, but a paired Evaluation->Prune veto (free-won't) is applied instead (key falsifier); - a landed write with no paired move is a durable no-step; - an illegal paired edge is surfaced (PersistError::Illegal), owner untouched. The DurableWrite seam is the durable-append surface; its concrete impl (next slice, a lance-having crate) wires lance's OFFICIAL MemWAL (dataset::mem_wal::WalAppender::append, Copyright The Lance Authors) over Arrow RecordBatch — hand-rolling no WAL, inventing nothing. Keeping it a trait leaves the ordering core lance-free / protoc-free (probe-first). Co-Authored-By: Claude --- crates/lance-graph-planner/src/lib.rs | 3 + .../lance-graph-planner/src/persist_sink.rs | 328 ++++++++++++++++++ 2 files changed, 331 insertions(+) create mode 100644 crates/lance-graph-planner/src/persist_sink.rs diff --git a/crates/lance-graph-planner/src/lib.rs b/crates/lance-graph-planner/src/lib.rs index 2605836a..77e51672 100644 --- a/crates/lance-graph-planner/src/lib.rs +++ b/crates/lance-graph-planner/src/lib.rs @@ -78,6 +78,9 @@ pub mod batch_writer; // === D-MBX-A6 Outcome→KanbanMove emit adapter (bootstrap rebind + ahead-cast) === pub mod owner_adapter; +// === D-MBX-A6 persistence-sink ordering core (post-write kanbanstep; DurableWrite seam) === +pub mod persist_sink; + // === Internal API (same-binary, zero-serde) === pub mod api; diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs new file mode 100644 index 00000000..ffb505e2 --- /dev/null +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -0,0 +1,328 @@ +//! The post-write kanbanstep — the ordering core of the D-MBX-A6 persistence sink. +//! +//! This is the POST-write half of the fire-and-forget flow (the pre-write half is +//! [`crate::owner_adapter`]). It runs on the **independent persistence path**, not +//! the thinker: by the time the sink drains a cast, the thinker has already +//! reported (`BatchWriter::cast`) and moved on. There is no ack, no callback into +//! the thinker, no confirmation ledger. +//! +//! ## The ordering invariant (operator-ruled — the thing the falsifier proves) +//! +//! ```text +//! durable write FIRST → the new LanceVersion IS the SoA's latest state +//! → ONLY THEN apply the cast's PAIRED move (the KanbanStep) +//! ``` +//! +//! - **No successful write ⇒ no step.** A write that does not land leaves the +//! owner's lifecycle untouched. +//! - **The step is the PAIRED move**, never a generic `next_phases().first()`. +//! The move the thought cast with this write (`paired.to`) is what the owner +//! advances to — a version appearing is not a licence to manufacture a generic +//! forward-arc tick. (Contrast [`crate::batch_writer`]'s note and the generic +//! `NextPhaseScheduler`, which is a *different*, legitimately-generic loop.) +//! - The owner's checked [`MailboxSoaOwner::try_advance_phase`] stays the SOLE +//! mutator; an illegal paired edge is surfaced ([`PersistError::Illegal`]), +//! never silently applied. +//! +//! ## The `DurableWrite` seam +//! +//! [`DurableWrite`] is the durable-append surface the sink drains into. Its +//! concrete implementation lives in a lance-having crate and wires Lance 7's +//! shipped MemWAL (`dataset::mem_wal::WalAppender::append(Vec)` via +//! `ShardWriter`) — it invents no persistence machinery. Keeping the durable +//! write behind this trait leaves the **ordering logic and its falsifier +//! lance-free and runnable without `protoc`**, which is the whole point of a +//! probe-first slice: prove the ordering before paying for the MemWAL build. + +use lance_graph_contract::collapse_gate::MailboxId; +use lance_graph_contract::kanban::{KanbanMove, RubiconTransitionError}; +use lance_graph_contract::scheduler::DatasetVersion; +use lance_graph_contract::soa_view::MailboxSoaOwner; + +/// A durable write that did not land (the WAL append failed / was fenced). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WriteFailed(pub String); + +/// The durable-append seam the persistence sink drains a cast into. +/// +/// The concrete impl (a lance-having crate) wires Lance 7's MemWAL +/// (`WalAppender::append`); this trait keeps the ordering core lance-free. +pub trait DurableWrite { + /// Durably append the owner's live SoA backing state (read zero-copy at flush + /// — the `bytes` are the resident `NodeRowPacket` view, never owned deltas) on + /// behalf of `owner`. + /// + /// `Ok(version)` = the write landed and that `LanceVersion` is now the SoA's + /// latest thinking state. `Err` = it did not land (⇒ the caller applies no + /// lifecycle step). + fn append(&mut self, owner: MailboxId, bytes: &[u8]) -> Result; +} + +/// Why a persist-then-step cycle produced no lifecycle advance. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PersistError { + /// The durable write did not land — **no lifecycle step was applied** and the + /// owner is untouched (the ordering invariant's negative half). + Write(WriteFailed), + /// The write landed, but the cast's paired move is not a legal Rubicon edge + /// from the owner's current phase — surfaced, never silently applied. + Illegal(RubiconTransitionError), +} + +/// The post-write kanbanstep: durably write the owner's SoA state, then — and +/// **only on a successful write** — apply the cast's PAIRED move. +/// +/// Returns: +/// - `Ok(Some(step))` — the write landed and the paired move was applied (the +/// `step` is the owner's own emitted [`KanbanMove`], carrying its live cycle); +/// - `Ok(None)` — the write landed but the cast carried no move (durable, no step); +/// - `Err(PersistError::Write)` — the write did not land (**no step**; owner untouched); +/// - `Err(PersistError::Illegal)` — the paired edge is illegal from the current +/// phase (surfaced; owner untouched). +/// +/// The applied transition target is `paired.to` — the move the thought cast with +/// this write — never a scheduler-manufactured generic successor. +pub fn persist_then_step( + owner: &mut O, + write: &mut W, + paired: Option, + bytes: &[u8], +) -> Result, PersistError> +where + W: DurableWrite, + O: MailboxSoaOwner, +{ + // 1. Durable write FIRST. If it does not land, return without touching the + // owner's lifecycle — no successful write ⇒ no step. + let _version = write + .append(owner.mailbox_id(), bytes) + .map_err(PersistError::Write)?; + // 2. Post-write: apply the PAIRED move's target via the checked airgap. Never + // a generic `next_phases().first()` — `paired.to` is the thought's intent. + match paired { + Some(mv) => owner + .try_advance_phase(mv.to) + .map(Some) + .map_err(PersistError::Illegal), + None => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lance_graph_contract::collapse_gate::MailboxId as MId; + use lance_graph_contract::kanban::{ExecTarget, KanbanColumn}; + use lance_graph_contract::soa_view::{MailboxSoaOwner, MailboxSoaView}; + + /// Minimal in-RAM owner (mirrors `kanban_actor::tests::TestBoard`). + struct FakeOwner { + id: MId, + phase: KanbanColumn, + cycle: u32, + } + impl MailboxSoaView for FakeOwner { + fn mailbox_id(&self) -> MId { + self.id + } + fn n_rows(&self) -> usize { + 0 + } + fn w_slot(&self) -> u8 { + 0 + } + fn current_cycle(&self) -> u32 { + self.cycle + } + fn phase(&self) -> KanbanColumn { + self.phase + } + fn energy(&self) -> &[f32] { + &[] + } + fn edges_raw(&self) -> &[u64] { + &[] + } + fn meta_raw(&self) -> &[u32] { + &[] + } + fn entity_type(&self) -> &[u16] { + &[] + } + } + impl MailboxSoaOwner for FakeOwner { + fn advance_phase(&mut self, to: KanbanColumn) -> KanbanMove { + let from = self.phase; + self.phase = to; + self.cycle = self.cycle.wrapping_add(1); + KanbanMove { + mailbox: self.id, + from, + to, + witness_chain_position: self.cycle, + exec: ExecTarget::Native, + } + } + } + + /// A `DurableWrite` whose success/failure and call-count are observable. + struct FakeWal { + succeed: bool, + version: u64, + calls: u32, + } + impl DurableWrite for FakeWal { + fn append(&mut self, _owner: MId, _bytes: &[u8]) -> Result { + self.calls += 1; + if self.succeed { + Ok(DatasetVersion(self.version)) + } else { + Err(WriteFailed("wal fenced".into())) + } + } + } + + fn owner(phase: KanbanColumn) -> FakeOwner { + FakeOwner { + id: 42, + phase, + cycle: 5, + } + } + fn paired(to: KanbanColumn) -> KanbanMove { + KanbanMove { + mailbox: 42, + from: KanbanColumn::Planning, + to, + witness_chain_position: 0, + exec: ExecTarget::Elixir, + } + } + + #[test] + fn no_successful_write_applies_no_step() { + // The negative half of the ordering invariant: a failed write leaves the + // owner's lifecycle UNTOUCHED. + let mut o = owner(KanbanColumn::Planning); + let mut w = FakeWal { + succeed: false, + version: 9, + calls: 0, + }; + let r = persist_then_step( + &mut o, + &mut w, + Some(paired(KanbanColumn::CognitiveWork)), + &[], + ); + assert_eq!( + r, + Err(PersistError::Write(WriteFailed("wal fenced".into()))) + ); + assert_eq!(w.calls, 1, "the write WAS attempted"); + // Anti-vacuity: the owner did NOT advance — no write, no step. + assert_eq!( + o.phase(), + KanbanColumn::Planning, + "no step on a failed write" + ); + assert_eq!(o.current_cycle(), 5, "cycle untouched"); + } + + #[test] + fn successful_write_then_applies_the_paired_move_once() { + let mut o = owner(KanbanColumn::Planning); + let mut w = FakeWal { + succeed: true, + version: 43, + calls: 0, + }; + let step = persist_then_step( + &mut o, + &mut w, + Some(paired(KanbanColumn::CognitiveWork)), + &[], + ) + .expect("write landed") + .expect("a paired move was applied"); + assert_eq!(step.to, KanbanColumn::CognitiveWork, "the paired target"); + assert_eq!( + o.phase(), + KanbanColumn::CognitiveWork, + "owner advanced once" + ); + assert_eq!(o.current_cycle(), 6, "exactly one advance"); + } + + #[test] + fn applies_the_paired_move_not_the_generic_successor() { + // THE key falsifier. From `Evaluation`, the generic forward arc + // (`next_phases().first()`) is `Commit`. But the thought cast a veto — + // the free-won't `Evaluation → Prune`. The sink must apply the PAIRED + // move (Prune), never the generic successor (Commit). + assert_eq!( + KanbanColumn::Evaluation.next_phases().first(), + Some(&KanbanColumn::Commit), + "precondition: generic successor is Commit", + ); + let mut o = owner(KanbanColumn::Evaluation); + let mut w = FakeWal { + succeed: true, + version: 44, + calls: 0, + }; + let step = persist_then_step(&mut o, &mut w, Some(paired(KanbanColumn::Prune)), &[]) + .expect("write landed") + .expect("paired veto applied"); + assert_eq!(step.to, KanbanColumn::Prune, "the paired veto, not Commit"); + assert_eq!(o.phase(), KanbanColumn::Prune); + assert_ne!( + o.phase(), + KanbanColumn::Commit, + "the generic successor was NOT taken", + ); + } + + #[test] + fn a_landed_write_with_no_paired_move_is_a_durable_no_step() { + // A write can land durably while carrying no lifecycle intent — durable, + // but no step (non-vacuous: the write WAS attempted and succeeded). + let mut o = owner(KanbanColumn::CognitiveWork); + let mut w = FakeWal { + succeed: true, + version: 7, + calls: 0, + }; + let r = persist_then_step(&mut o, &mut w, None, &[]); + assert_eq!(r, Ok(None), "durable write, no step"); + assert_eq!(w.calls, 1, "the write did land"); + assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "phase unchanged"); + } + + #[test] + fn an_illegal_paired_edge_is_surfaced_not_applied() { + // The write lands, but the paired move is an illegal Rubicon edge from the + // current phase (Planning → Evaluation skips CognitiveWork). Surfaced as + // Illegal; the owner is NOT mutated (the checked airgap holds post-write). + let mut o = owner(KanbanColumn::Planning); + let mut w = FakeWal { + succeed: true, + version: 8, + calls: 0, + }; + let r = persist_then_step(&mut o, &mut w, Some(paired(KanbanColumn::Evaluation)), &[]); + assert!( + matches!(r, Err(PersistError::Illegal(_))), + "illegal edge surfaced" + ); + assert_eq!( + o.phase(), + KanbanColumn::Planning, + "owner untouched on illegal edge" + ); + assert_eq!( + w.calls, 1, + "the write still landed (durable) before the rejected step" + ); + } +} From bb5fcdd8e08c9d7856df8d6dcc5c4e0c81bfca59 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 22:41:56 +0000 Subject: [PATCH 2/9] refactor(planner): split the persistence sink into two clock domains (operator ruling) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A monolithic persist_then_step(&mut owner, ..., bytes).await would hold &mut owner AND owner-borrowed bytes across the WAL I/O await — immobilizing the owner during object-store latency and defeating fire-and-forget. Split into: - persist_cast(sink, PersistCast) -> DurableReceipt (async; NO owner in the signature — never borrowed across the await; the owner stays free while durability runs); - apply_durable_step(&mut owner, DurableReceipt) (sync; NO await; the exclusive owner borrow is held only here, outside the storage-latency window). Durability type corrected: a WAL append yields a DurableCoordinate (shard + writer_epoch + wal_entry_position), NOT a DatasetVersion (that arrives later via MemTable flush + manifest commit). DurableReceipt carries the coordinate + the paired move; a receipt is refused for a foreign owner (OwnerMismatch). 7 probes green: success->receipt, failure->no receipt (=> no step reachable), paired-move applied, paired!=generic (Evaluation->Prune veto not Commit), no-move no-step, illegal edge surfaced, foreign-owner refused. Concrete impl (next): lance 7 ShardWriter::put (enable_memtable + durable_write) over an Arrow RecordBatch with shared-buffer ownership. Co-Authored-By: Claude --- .../lance-graph-planner/src/persist_sink.rs | 441 +++++++++++------- 1 file changed, 266 insertions(+), 175 deletions(-) diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs index ffb505e2..080af796 100644 --- a/crates/lance-graph-planner/src/persist_sink.rs +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -1,105 +1,176 @@ -//! The post-write kanbanstep — the ordering core of the D-MBX-A6 persistence sink. +//! The D-MBX-A6 persistence sink — **two cleanly separated clock domains**. //! -//! This is the POST-write half of the fire-and-forget flow (the pre-write half is -//! [`crate::owner_adapter`]). It runs on the **independent persistence path**, not -//! the thinker: by the time the sink drains a cast, the thinker has already -//! reported (`BatchWriter::cast`) and moved on. There is no ack, no callback into -//! the thinker, no confirmation ledger. -//! -//! ## The ordering invariant (operator-ruled — the thing the falsifier proves) +//! This is the POST-write half of the fire-and-forget flow (pre-write half = +//! [`crate::owner_adapter`]). The thinker already reported and moved on: //! //! ```text -//! durable write FIRST → the new LanceVersion IS the SoA's latest state -//! → ONLY THEN apply the cast's PAIRED move (the KanbanStep) +//! thinking path: SoA → BatchWriter.cast(on_behalf, paired_move, descriptor) → keep thinking +//! persistence path: PersistCast → async durable append → DurableReceipt (NO owner borrow) +//! owner-local step: DurableReceipt + paired move → try_advance_phase (NO await) //! ``` //! -//! - **No successful write ⇒ no step.** A write that does not land leaves the -//! owner's lifecycle untouched. -//! - **The step is the PAIRED move**, never a generic `next_phases().first()`. -//! The move the thought cast with this write (`paired.to`) is what the owner -//! advances to — a version appearing is not a licence to manufacture a generic -//! forward-arc tick. (Contrast [`crate::batch_writer`]'s note and the generic -//! `NextPhaseScheduler`, which is a *different*, legitimately-generic loop.) -//! - The owner's checked [`MailboxSoaOwner::try_advance_phase`] stays the SOLE -//! mutator; an illegal paired edge is surfaced ([`PersistError::Illegal`]), -//! never silently applied. +//! ## Why the two phases must NOT be one async function (operator-ruled) +//! +//! A monolithic `persist_then_step(&mut owner, …, bytes).await` would hold BOTH +//! `&mut owner` and `&[u8]` borrowed from live owner state **across the WAL I/O +//! await** — immobilizing the owner while the object store completes, which +//! defeats the latency masking the architecture exists to obtain. So the durable +//! write ([`persist_cast`], async, **no `O` in its signature**) and the lifecycle +//! step ([`apply_durable_step`], sync, **no await**) are split. The exclusive +//! `&mut owner` is held only in the sync phase, outside the storage-latency window; +//! `MailboxSoaOwner::try_advance_phase` stays the SOLE lifecycle mutator. +//! +//! ## The ordering invariant (what the falsifiers prove) +//! +//! - **No durable write ⇒ no receipt ⇒ no step.** A failed append yields no +//! [`DurableReceipt`], and [`apply_durable_step`] cannot be reached without one. +//! - **The step is the PAIRED move** (`receipt.paired_move.to`), never a generic +//! `next_phases().first()`. The move the thought cast rides in the receipt. +//! - A receipt is applied only to **its own** owner ([`PersistError::OwnerMismatch`]). //! -//! ## The `DurableWrite` seam +//! ## The `DurableWrite` seam + the durability type //! -//! [`DurableWrite`] is the durable-append surface the sink drains into. Its -//! concrete implementation lives in a lance-having crate and wires Lance 7's -//! shipped MemWAL (`dataset::mem_wal::WalAppender::append(Vec)` via -//! `ShardWriter`) — it invents no persistence machinery. Keeping the durable -//! write behind this trait leaves the **ordering logic and its falsifier -//! lance-free and runnable without `protoc`**, which is the whole point of a -//! probe-first slice: prove the ordering before paying for the MemWAL build. +//! A WAL append produces a durable **coordinate** (shard + writer epoch + WAL +//! entry position), NOT a base `DatasetVersion` — the dataset version arrives +//! later via MemTable flush + manifest commit. So [`DurableWrite::append`] returns +//! [`DurableCoordinate`], never a version. The concrete impl (a lance-having +//! crate) wires lance 7.0.0's OFFICIAL MemWAL — preferring the high-level +//! `ShardWriter::put` (`enable_memtable + durable_write`: insert into the +//! queryable MemTable, release the lock, then await WAL durability off-lock) over +//! the raw `WalAppender::append` primitive — over an Arrow `RecordBatch` whose +//! buffers are independently owned (Arrow shared-buffer ownership), so "zero-copy" +//! never means holding the SoA owner borrowed across I/O. +//! +//! Keeping the seam a trait leaves the ordering core lance-free and `protoc`-free. use lance_graph_contract::collapse_gate::MailboxId; use lance_graph_contract::kanban::{KanbanMove, RubiconTransitionError}; -use lance_graph_contract::scheduler::DatasetVersion; use lance_graph_contract::soa_view::MailboxSoaOwner; /// A durable write that did not land (the WAL append failed / was fenced). #[derive(Debug, Clone, PartialEq, Eq)] pub struct WriteFailed(pub String); -/// The durable-append seam the persistence sink drains a cast into. +/// A backend-neutral **durable coordinate** — where the write landed in the +/// durable log. This is NOT a base Lance `DatasetVersion` (that arrives later via +/// MemTable flush + manifest commit); it is the WAL/LSM coordinate that proves +/// durability now. For lance: `shard` = shard `Uuid` (as `u128`), +/// `writer_epoch` + `wal_entry_position` from the `ShardWriter`/`WalAppendResult`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DurableCoordinate { + /// Opaque shard identity (lance: the shard `Uuid` as `u128`). + pub shard: u128, + /// The writer epoch that fenced this append. + pub writer_epoch: u64, + /// The monotonic WAL entry position of this durable append. + pub wal_entry_position: u64, +} + +/// The async durable-append seam. /// -/// The concrete impl (a lance-having crate) wires Lance 7's MemWAL -/// (`WalAppender::append`); this trait keeps the ordering core lance-free. +/// `&self` (shared — many casts drain concurrently) and **no owner borrow**: the +/// persistence path must not hold the SoA owner across object-store I/O. `payload` +/// is an independently-owned buffer (never a borrow of live owner state), so the +/// owner stays free while the WAL hums. `async` because lance's WAL is async and +/// the sink runs on the background persistence path (never the hot thinker path); +/// `async_fn_in_trait` is allowed (generic use only, never `dyn`). +#[allow(async_fn_in_trait)] pub trait DurableWrite { - /// Durably append the owner's live SoA backing state (read zero-copy at flush - /// — the `bytes` are the resident `NodeRowPacket` view, never owned deltas) on - /// behalf of `owner`. - /// - /// `Ok(version)` = the write landed and that `LanceVersion` is now the SoA's - /// latest thinking state. `Err` = it did not land (⇒ the caller applies no - /// lifecycle step). - fn append(&mut self, owner: MailboxId, bytes: &[u8]) -> Result; + /// Durably append `payload` on behalf of `owner`. `Ok(coordinate)` = it landed + /// (durable now); `Err` = it did not (⇒ no receipt, ⇒ no step). + async fn append( + &self, + owner: MailboxId, + payload: &[u8], + ) -> Result; } -/// Why a persist-then-step cycle produced no lifecycle advance. +/// A **detached** cast envelope — the thinker's report, carrying its OWN payload +/// (independently owned, never a borrow of live owner state) so persistence runs +/// with the owner free. Mirrors what `BatchWriter::cast(on_behalf, moves, payload)` +/// stages; at drain the descriptor is resolved to `payload` bytes. +pub struct PersistCast { + /// The mailbox this write is on behalf of. + pub owner: MailboxId, + /// The lifecycle move the thought cast with this write (its `to` is applied + /// post-durability). `None` when the cast carries no lifecycle intent. + pub paired_move: Option, + /// The bytes to persist — an owned/independent buffer (the concrete sink forms + /// an Arrow `RecordBatch` over it with shared-buffer ownership). + pub payload: Vec, +} + +/// Proof the write landed (a [`DurableCoordinate`]) plus the paired move to apply +/// and the owner it belongs to. Produced by [`persist_cast`] on success; consumed +/// by [`apply_durable_step`]. Its mere existence is the "the write landed" fact — +/// there is no separate ack/confirmation ledger (`E-ACK-ELIMINATED-1`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DurableReceipt { + /// The mailbox the durable write was on behalf of. + pub owner: MailboxId, + /// The paired lifecycle move to apply post-durability. + pub paired_move: Option, + /// Where the write landed in the durable log. + pub coordinate: DurableCoordinate, +} + +/// Why a persist / step operation produced no lifecycle advance. #[derive(Debug, Clone, PartialEq, Eq)] pub enum PersistError { - /// The durable write did not land — **no lifecycle step was applied** and the - /// owner is untouched (the ordering invariant's negative half). + /// The durable write did not land — **no receipt exists, so no step can be + /// applied** (the ordering invariant's negative half). Write(WriteFailed), - /// The write landed, but the cast's paired move is not a legal Rubicon edge - /// from the owner's current phase — surfaced, never silently applied. + /// The paired move is not a legal Rubicon edge from the owner's current phase — + /// surfaced, never silently applied. Illegal(RubiconTransitionError), + /// A receipt was applied to the wrong owner — refused (a receipt only advances + /// the mailbox it was minted for; write-on-behalf never crosses owners). + OwnerMismatch { + receipt_owner: MailboxId, + applying_to: MailboxId, + }, } -/// The post-write kanbanstep: durably write the owner's SoA state, then — and -/// **only on a successful write** — apply the cast's PAIRED move. -/// -/// Returns: -/// - `Ok(Some(step))` — the write landed and the paired move was applied (the -/// `step` is the owner's own emitted [`KanbanMove`], carrying its live cycle); -/// - `Ok(None)` — the write landed but the cast carried no move (durable, no step); -/// - `Err(PersistError::Write)` — the write did not land (**no step**; owner untouched); -/// - `Err(PersistError::Illegal)` — the paired edge is illegal from the current -/// phase (surfaced; owner untouched). +/// **Phase 1 — async persistence, NO owner borrow.** Durably append the cast's +/// payload; on success return a [`DurableReceipt`], on failure a +/// [`PersistError::Write`] and **no receipt**. `O` (the owner) does not appear in +/// this signature at all — it is never borrowed across the WAL / object-store +/// await, so the owner stays free while durability runs. +pub async fn persist_cast( + sink: &W, + cast: PersistCast, +) -> Result { + let coordinate = sink + .append(cast.owner, &cast.payload) + .await + .map_err(PersistError::Write)?; + Ok(DurableReceipt { + owner: cast.owner, + paired_move: cast.paired_move, + coordinate, + }) +} + +/// **Phase 2 — synchronous owner-local completion, NO await.** Given a +/// [`DurableReceipt`] (⇒ the write already landed), apply the PAIRED move via the +/// owner's checked `try_advance_phase`. The exclusive `&mut owner` is held only +/// here, outside the storage-latency window. Refuses a receipt minted for a +/// different owner ([`PersistError::OwnerMismatch`]). /// -/// The applied transition target is `paired.to` — the move the thought cast with -/// this write — never a scheduler-manufactured generic successor. -pub fn persist_then_step( +/// Returns `Ok(Some(step))` (paired move applied), `Ok(None)` (receipt carried no +/// move — a durable no-step), or an error. The transition target is +/// `receipt.paired_move.to` — never a generic successor. +pub fn apply_durable_step( owner: &mut O, - write: &mut W, - paired: Option, - bytes: &[u8], -) -> Result, PersistError> -where - W: DurableWrite, - O: MailboxSoaOwner, -{ - // 1. Durable write FIRST. If it does not land, return without touching the - // owner's lifecycle — no successful write ⇒ no step. - let _version = write - .append(owner.mailbox_id(), bytes) - .map_err(PersistError::Write)?; - // 2. Post-write: apply the PAIRED move's target via the checked airgap. Never - // a generic `next_phases().first()` — `paired.to` is the thought's intent. - match paired { + receipt: DurableReceipt, +) -> Result, PersistError> { + if receipt.owner != owner.mailbox_id() { + return Err(PersistError::OwnerMismatch { + receipt_owner: receipt.owner, + applying_to: owner.mailbox_id(), + }); + } + match receipt.paired_move { Some(mv) => owner .try_advance_phase(mv.to) .map(Some) @@ -166,16 +237,32 @@ mod tests { } /// A `DurableWrite` whose success/failure and call-count are observable. - struct FakeWal { + /// `&self` + interior-mutable counter, mirroring the real sink's shared shape. + struct FakeSink { succeed: bool, - version: u64, - calls: u32, + calls: std::cell::Cell, + } + impl FakeSink { + fn new(succeed: bool) -> Self { + Self { + succeed, + calls: std::cell::Cell::new(0), + } + } } - impl DurableWrite for FakeWal { - fn append(&mut self, _owner: MId, _bytes: &[u8]) -> Result { - self.calls += 1; + impl DurableWrite for FakeSink { + async fn append( + &self, + _owner: MId, + _payload: &[u8], + ) -> Result { + self.calls.set(self.calls.get() + 1); if self.succeed { - Ok(DatasetVersion(self.version)) + Ok(DurableCoordinate { + shard: 0xABCD, + writer_epoch: 1, + wal_entry_position: 7, + }) } else { Err(WriteFailed("wal fenced".into())) } @@ -189,140 +276,144 @@ mod tests { cycle: 5, } } - fn paired(to: KanbanColumn) -> KanbanMove { - KanbanMove { - mailbox: 42, - from: KanbanColumn::Planning, - to, - witness_chain_position: 0, - exec: ExecTarget::Elixir, + fn cast(paired_to: Option) -> PersistCast { + PersistCast { + owner: 42, + paired_move: paired_to.map(|to| KanbanMove { + mailbox: 42, + from: KanbanColumn::Planning, + to, + witness_chain_position: 0, + exec: ExecTarget::Elixir, + }), + payload: vec![1, 2, 3, 4], + } + } + fn receipt_for(paired_to: Option) -> DurableReceipt { + DurableReceipt { + owner: 42, + paired_move: paired_to.map(|to| KanbanMove { + mailbox: 42, + from: KanbanColumn::Planning, + to, + witness_chain_position: 0, + exec: ExecTarget::Elixir, + }), + coordinate: DurableCoordinate { + shard: 0xABCD, + writer_epoch: 1, + wal_entry_position: 7, + }, } } - #[test] - fn no_successful_write_applies_no_step() { - // The negative half of the ordering invariant: a failed write leaves the - // owner's lifecycle UNTOUCHED. - let mut o = owner(KanbanColumn::Planning); - let mut w = FakeWal { - succeed: false, - version: 9, - calls: 0, - }; - let r = persist_then_step( - &mut o, - &mut w, - Some(paired(KanbanColumn::CognitiveWork)), - &[], + // ── Phase 1: async persistence, NO owner borrow ──────────────────────────── + + #[tokio::test] + async fn a_successful_write_yields_a_receipt_carrying_the_paired_move() { + let sink = FakeSink::new(true); + let r = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))) + .await + .expect("write landed"); + assert_eq!(r.owner, 42); + assert_eq!( + r.paired_move.map(|m| m.to), + Some(KanbanColumn::CognitiveWork) ); assert_eq!( - r, - Err(PersistError::Write(WriteFailed("wal fenced".into()))) + r.coordinate.wal_entry_position, 7, + "the durable coordinate rides" ); - assert_eq!(w.calls, 1, "the write WAS attempted"); - // Anti-vacuity: the owner did NOT advance — no write, no step. + assert_eq!(sink.calls.get(), 1, "the write was attempted once"); + } + + #[tokio::test] + async fn a_failed_write_yields_no_receipt() { + // The negative half: no durable write ⇒ NO receipt exists. There is + // therefore nothing to hand to apply_durable_step ⇒ no step can occur. + let sink = FakeSink::new(false); + let r = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; assert_eq!( - o.phase(), - KanbanColumn::Planning, - "no step on a failed write" + r, + Err(PersistError::Write(WriteFailed("wal fenced".into()))) ); - assert_eq!(o.current_cycle(), 5, "cycle untouched"); + assert_eq!(sink.calls.get(), 1, "the write was attempted"); } + // ── Phase 2: synchronous owner-local completion, NO await ─────────────────── + #[test] - fn successful_write_then_applies_the_paired_move_once() { + fn applying_a_receipt_advances_the_owner_by_the_paired_move() { let mut o = owner(KanbanColumn::Planning); - let mut w = FakeWal { - succeed: true, - version: 43, - calls: 0, - }; - let step = persist_then_step( - &mut o, - &mut w, - Some(paired(KanbanColumn::CognitiveWork)), - &[], - ) - .expect("write landed") - .expect("a paired move was applied"); - assert_eq!(step.to, KanbanColumn::CognitiveWork, "the paired target"); - assert_eq!( - o.phase(), - KanbanColumn::CognitiveWork, - "owner advanced once" - ); - assert_eq!(o.current_cycle(), 6, "exactly one advance"); + let step = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::CognitiveWork))) + .expect("legal") + .expect("a paired move"); + assert_eq!(step.to, KanbanColumn::CognitiveWork); + assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "advanced once"); + assert_eq!(o.current_cycle(), 6); } #[test] - fn applies_the_paired_move_not_the_generic_successor() { - // THE key falsifier. From `Evaluation`, the generic forward arc - // (`next_phases().first()`) is `Commit`. But the thought cast a veto — - // the free-won't `Evaluation → Prune`. The sink must apply the PAIRED - // move (Prune), never the generic successor (Commit). + fn applying_a_receipt_uses_the_paired_move_not_the_generic_successor() { + // THE key falsifier. From `Evaluation` the generic forward arc is `Commit`; + // the receipt carries the free-won't veto `Evaluation → Prune`. The step + // must be the PAIRED move (Prune), never the generic successor (Commit). assert_eq!( KanbanColumn::Evaluation.next_phases().first(), Some(&KanbanColumn::Commit), "precondition: generic successor is Commit", ); let mut o = owner(KanbanColumn::Evaluation); - let mut w = FakeWal { - succeed: true, - version: 44, - calls: 0, - }; - let step = persist_then_step(&mut o, &mut w, Some(paired(KanbanColumn::Prune)), &[]) - .expect("write landed") - .expect("paired veto applied"); + let step = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::Prune))) + .expect("legal") + .expect("paired veto"); assert_eq!(step.to, KanbanColumn::Prune, "the paired veto, not Commit"); assert_eq!(o.phase(), KanbanColumn::Prune); assert_ne!( o.phase(), KanbanColumn::Commit, - "the generic successor was NOT taken", + "generic successor NOT taken" ); } #[test] - fn a_landed_write_with_no_paired_move_is_a_durable_no_step() { - // A write can land durably while carrying no lifecycle intent — durable, - // but no step (non-vacuous: the write WAS attempted and succeeded). + fn a_receipt_with_no_paired_move_is_a_durable_no_step() { let mut o = owner(KanbanColumn::CognitiveWork); - let mut w = FakeWal { - succeed: true, - version: 7, - calls: 0, - }; - let r = persist_then_step(&mut o, &mut w, None, &[]); - assert_eq!(r, Ok(None), "durable write, no step"); - assert_eq!(w.calls, 1, "the write did land"); + let r = apply_durable_step(&mut o, receipt_for(None)); + assert_eq!(r, Ok(None), "durable, but no step"); assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "phase unchanged"); } #[test] fn an_illegal_paired_edge_is_surfaced_not_applied() { - // The write lands, but the paired move is an illegal Rubicon edge from the - // current phase (Planning → Evaluation skips CognitiveWork). Surfaced as - // Illegal; the owner is NOT mutated (the checked airgap holds post-write). + // Planning → Evaluation skips CognitiveWork — illegal. Surfaced; owner + // untouched (the checked airgap holds in the owner-local phase). let mut o = owner(KanbanColumn::Planning); - let mut w = FakeWal { - succeed: true, - version: 8, - calls: 0, + let r = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::Evaluation))); + assert!(matches!(r, Err(PersistError::Illegal(_)))); + assert_eq!(o.phase(), KanbanColumn::Planning, "owner untouched"); + } + + #[test] + fn a_receipt_is_refused_for_a_foreign_owner() { + // A receipt minted for mailbox 42 must never advance mailbox 99. + let mut other = FakeOwner { + id: 99, + phase: KanbanColumn::Planning, + cycle: 5, }; - let r = persist_then_step(&mut o, &mut w, Some(paired(KanbanColumn::Evaluation)), &[]); - assert!( - matches!(r, Err(PersistError::Illegal(_))), - "illegal edge surfaced" - ); + let r = apply_durable_step(&mut other, receipt_for(Some(KanbanColumn::CognitiveWork))); + assert!(matches!( + r, + Err(PersistError::OwnerMismatch { + receipt_owner: 42, + applying_to: 99 + }) + )); assert_eq!( - o.phase(), + other.phase(), KanbanColumn::Planning, - "owner untouched on illegal edge" - ); - assert_eq!( - w.calls, 1, - "the write still landed (durable) before the rejected step" + "foreign owner untouched" ); } } From 8dcfbbe94ed1fa596b941f11cc8ea416280c56a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 23:29:27 +0000 Subject: [PATCH 3/9] feat(planner): durable-witness reshape + temporal layer-1 causal deinterlacing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the crash-durability gap the operator named on the persistence sink, and wires temporal.rs layer-1 — both mandated BEFORE any concrete LanceShardSink. ## The gap (persist_sink) At the prior split-phase shape, persist_cast appended only the payload; the paired move rode solely in the in-memory DurableReceipt. WAL append lands → process dies before apply_durable_step → the receipt evaporates → the paired move is lost → the KanbanStep never fires, though the durable SoA state moved. Silent: storage advanced, lifecycle did not. ## The reshape - DurableWitness (owner, cast_id, cycle, paired_move) is CO-LOCATED with the SoA payload in ONE persistence generation: DurableWrite::append(&witness, &payload). The witness is the crash-durable copy of the move, never in-memory-only. - DurableReceipt now merely REFERENCES that durable material (via its DurableCoordinate); it is not the only copy of the move. - DurableWrite::scan_witnesses replay seam reads the co-located witnesses back — a read of what durable storage already holds, NOT a separate ack ledger. - recover_and_apply(owner, witnesses): crash recovery reads the witnesses, runs temporal layer-1 to find the owner's pending tail, and replays in cast order; stale moves (already reflected in the recovered state) are skipped (idempotent). - apply_durable_step gains a from==phase guard (PersistError::StalePhase) so a stale/out-of-order apply on the synchronous path is surfaced loudly. Durability proof stays the DurableCoordinate (shard/epoch/wal position), never a LanceVersion — the write is durable before any base manifest version attaches. ## temporal.rs layer-1 — CAUSAL deinterlacing New LocalCausalRow trait (owner + cast_seq) + local_trajectories / local_trajectory_of split a globally-interleaved durable log into per-owner LOCAL causal chains, ordered by cast_seq. A@s0,C@s0,B@s0,A@s1 → owner A's chain [A@s0,A@s1]; the interleaved owners are REMOVED from A's timeline. This is the layer-1 counterpart to the existing layer-2 epistemic projection (classify / deinterlace); the two compose (layer-1 reconstructs the chain, layer-2 filters it by the reader's horizon). DurableWitness implements LocalCausalRow. ## Falsifiers (all green — the operator's five integration falsifiers) 1. layer-1 deinterlaces interleaved global log → owner-local chain (+ vacuity guard: other owners' rows dropped, not reordered) 2. one owner's batch replays in cast order (deliberately seq-descending log) 3. WAL succeeds + crash (receipt dropped) → recovery reconstructs+applies the move from the durable witness; idempotent re-run applies nothing 4. WAL fails → no witness lands → nothing to replay → no move, no step 5. WAL-visible-before-manifest → recovery identifies latest local state from cast order, no dataset version exists 343 planner lib tests pass; clippy clean; fmt clean. Builds NO concrete sink. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .../lance-graph-planner/src/persist_sink.rs | 435 ++++++++++++++++-- crates/lance-graph-planner/src/temporal.rs | 198 ++++++++ 2 files changed, 592 insertions(+), 41 deletions(-) diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs index 080af796..2e48bffa 100644 --- a/crates/lance-graph-planner/src/persist_sink.rs +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -28,25 +28,58 @@ //! `next_phases().first()`. The move the thought cast rides in the receipt. //! - A receipt is applied only to **its own** owner ([`PersistError::OwnerMismatch`]). //! +//! ## Crash-durability: the paired move is CO-LOCATED, not in-memory-only +//! +//! The paired transition witness is **durably co-located with the SoA state in +//! the same persistence generation** (operator ruling). The naïve split persists +//! only the payload and keeps the paired move alive solely in the in-memory +//! [`DurableReceipt`]; then *WAL append lands → process dies before +//! [`apply_durable_step`] → the receipt evaporates → the paired move is lost → +//! the KanbanStep never fires*, even though the durable SoA state moved. The gap +//! is silent: storage advanced, lifecycle did not. +//! +//! So [`DurableWrite::append`] takes the [`DurableWitness`] (owner, cast id, +//! cycle, paired move) **alongside** the payload and lands BOTH atomically. The +//! [`DurableReceipt`] then merely *references* that durable material (via its +//! [`DurableCoordinate`]) for the fast in-process apply; it is **not the only +//! copy**. On restart, [`recover_and_apply`] reads the witnesses back +//! ([`DurableWrite::scan_witnesses`]), runs `temporal` layer-1 causal +//! deinterlacing to find each owner's pending tail, and re-applies the moves in +//! cast order. This is a read of what durable storage already holds — **not** a +//! separate ack / confirmation ledger (`E-ACK-ELIMINATED-1`). +//! //! ## The `DurableWrite` seam + the durability type //! //! A WAL append produces a durable **coordinate** (shard + writer epoch + WAL //! entry position), NOT a base `DatasetVersion` — the dataset version arrives //! later via MemTable flush + manifest commit. So [`DurableWrite::append`] returns -//! [`DurableCoordinate`], never a version. The concrete impl (a lance-having -//! crate) wires lance 7.0.0's OFFICIAL MemWAL — preferring the high-level -//! `ShardWriter::put` (`enable_memtable + durable_write`: insert into the -//! queryable MemTable, release the lock, then await WAL durability off-lock) over -//! the raw `WalAppender::append` primitive — over an Arrow `RecordBatch` whose -//! buffers are independently owned (Arrow shared-buffer ownership), so "zero-copy" -//! never means holding the SoA owner borrowed across I/O. +//! [`DurableCoordinate`], never a version. This is exactly why the coordinate, +//! not `LanceVersion`, is the durability proof: the write is queryable and +//! durable *before* any base manifest version attaches (falsifier 5 — +//! WAL-visible-before-manifest — proves the latest local state is identified from +//! the co-located witness's cast order, not from a dataset version that does not +//! exist yet). +//! +//! The concrete impl (a lance-having crate) wires lance 7.0.0's OFFICIAL MemWAL — +//! preferring the high-level `ShardWriter::put` (`enable_memtable + durable_write`: +//! insert into the queryable MemTable, release the lock, then await WAL durability +//! off-lock) over the raw `WalAppender::append` primitive. The witness and the +//! payload are two columns of ONE `RecordBatch` (one generation, atomic), whose +//! buffers are independently owned, so "no owner borrow across I/O" holds without +//! any claim of magic zero-copy. //! //! Keeping the seam a trait leaves the ordering core lance-free and `protoc`-free. +//! **This module builds NO concrete `LanceShardSink`** — per operator ruling the +//! durable-witness reshape + `temporal` layer-1 land first; the production sink +//! comes after, gated on the crash-recovery falsifiers here going green. use lance_graph_contract::collapse_gate::MailboxId; -use lance_graph_contract::kanban::{KanbanMove, RubiconTransitionError}; +use lance_graph_contract::kanban::{KanbanColumn, KanbanMove, RubiconTransitionError}; use lance_graph_contract::soa_view::MailboxSoaOwner; +use crate::batch_writer::CastId; +use crate::temporal::{local_trajectory_of, LocalCausalRow}; + /// A durable write that did not land (the WAL append failed / was fenced). #[derive(Debug, Clone, PartialEq, Eq)] pub struct WriteFailed(pub String); @@ -66,23 +99,65 @@ pub struct DurableCoordinate { pub wal_entry_position: u64, } -/// The async durable-append seam. +/// The durable transition witness — CO-LOCATED with the SoA payload in the same +/// persistence generation so it survives a crash (module doc § Crash-durability). +/// +/// Everything needed to re-apply a pending KanbanStep after a restart lives here: +/// WHO (`owner`), WHICH cast (`cast_id` — the owner-local replay order), the +/// owner-local `cycle`, and the `paired_move` to apply. On recovery, +/// [`recover_and_apply`] reads these back and replays in `cast_id` order. +/// +/// Implements [`LocalCausalRow`] so `temporal` layer-1 can deinterlace a global +/// interleaved witness stream into each owner's local trajectory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DurableWitness { + /// The mailbox this write is on behalf of — the deinterlace grouping key. + pub owner: MailboxId, + /// The cast this write realises — its `.0` is the owner-local replay order. + pub cast_id: CastId, + /// The owner-local cycle at the time of the cast (audit / trajectory). + pub cycle: u32, + /// The lifecycle move to apply once this generation is durable. `None` when + /// the cast carried no lifecycle intent (a durable no-step). + pub paired_move: Option, +} + +impl LocalCausalRow for DurableWitness { + fn owner(&self) -> MailboxId { + self.owner + } + fn cast_seq(&self) -> u64 { + self.cast_id.0 + } +} + +/// The async durable-append + replay seam. /// /// `&self` (shared — many casts drain concurrently) and **no owner borrow**: the -/// persistence path must not hold the SoA owner across object-store I/O. `payload` -/// is an independently-owned buffer (never a borrow of live owner state), so the -/// owner stays free while the WAL hums. `async` because lance's WAL is async and -/// the sink runs on the background persistence path (never the hot thinker path); -/// `async_fn_in_trait` is allowed (generic use only, never `dyn`). +/// persistence path must not hold the SoA owner across object-store I/O. Both the +/// [`DurableWitness`] and the `payload` are independently owned (never a borrow of +/// live owner state), so the owner stays free while the WAL hums. `async` because +/// lance's WAL is async and the sink runs on the background persistence path +/// (never the hot thinker path); `async_fn_in_trait` is allowed (generic use +/// only, never `dyn`). #[allow(async_fn_in_trait)] pub trait DurableWrite { - /// Durably append `payload` on behalf of `owner`. `Ok(coordinate)` = it landed - /// (durable now); `Err` = it did not (⇒ no receipt, ⇒ no step). + /// Durably append the `witness` CO-LOCATED with `payload`, atomically, in ONE + /// persistence generation. `Ok(coordinate)` = both landed together (durable + /// now); `Err` = neither did (⇒ no receipt, ⇒ no step). The witness is the + /// crash-durable copy of the paired move — never in-memory-only. async fn append( &self, - owner: MailboxId, + witness: &DurableWitness, payload: &[u8], ) -> Result; + + /// Replay seam: read back every durably-landed [`DurableWitness`], in the + /// durable log's own (globally interleaved) order. Crash recovery scans this, + /// runs `temporal` layer-1 to split it per owner, and re-applies pending moves + /// in cast order ([`recover_and_apply`]). Reading the SAME co-located material + /// the receipts merely referenced — NOT a separate ledger. + async fn scan_witnesses(&self) -> Result, WriteFailed>; } /// A **detached** cast envelope — the thinker's report, carrying its OWN payload @@ -92,11 +167,16 @@ pub trait DurableWrite { pub struct PersistCast { /// The mailbox this write is on behalf of. pub owner: MailboxId, + /// The cast this write realises — the owner-local replay order (from + /// `BatchWriter::cast`). Rides into the [`DurableWitness`] for crash replay. + pub cast_id: CastId, + /// The owner-local cycle at cast time — rides into the witness (audit). + pub cycle: u32, /// The lifecycle move the thought cast with this write (its `to` is applied /// post-durability). `None` when the cast carries no lifecycle intent. pub paired_move: Option, /// The bytes to persist — an owned/independent buffer (the concrete sink forms - /// an Arrow `RecordBatch` over it with shared-buffer ownership). + /// one Arrow `RecordBatch` with the witness as a second column, one generation). pub payload: Vec, } @@ -104,13 +184,25 @@ pub struct PersistCast { /// and the owner it belongs to. Produced by [`persist_cast`] on success; consumed /// by [`apply_durable_step`]. Its mere existence is the "the write landed" fact — /// there is no separate ack/confirmation ledger (`E-ACK-ELIMINATED-1`). +/// +/// **This receipt REFERENCES durable material; it is not the only copy of the +/// paired move.** The move is co-located in the durable generation the +/// [`coordinate`](Self::coordinate) points at (via the [`DurableWitness`] that +/// [`persist_cast`] appended). If this in-memory receipt is lost to a crash +/// before [`apply_durable_step`] runs, [`recover_and_apply`] reconstructs the +/// move from that durable generation — the KanbanStep is not lost with the +/// process. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DurableReceipt { /// The mailbox the durable write was on behalf of. pub owner: MailboxId, - /// The paired lifecycle move to apply post-durability. + /// The cast this receipt realises — mirrors the co-located witness's cast id. + pub cast_id: CastId, + /// The paired lifecycle move to apply post-durability. A convenience copy of + /// the co-located witness's move (see the type doc — durable, not only here). pub paired_move: Option, - /// Where the write landed in the durable log. + /// Where the write landed in the durable log — the reference INTO the durable + /// generation that co-locates the witness with the SoA state. pub coordinate: DurableCoordinate, } @@ -129,23 +221,41 @@ pub enum PersistError { receipt_owner: MailboxId, applying_to: MailboxId, }, + /// The paired move's `from` does not match the owner's current phase — the + /// move is stale (already applied, or out of cast order). Surfaced on the + /// synchronous single-receipt path ([`apply_durable_step`]) so a mis-ordered + /// or double apply is loud; the crash-recovery path ([`recover_and_apply`]) + /// instead SKIPS a stale move (it is already reflected in the durable state). + StalePhase { + owner_phase: KanbanColumn, + move_from: KanbanColumn, + }, } -/// **Phase 1 — async persistence, NO owner borrow.** Durably append the cast's -/// payload; on success return a [`DurableReceipt`], on failure a -/// [`PersistError::Write`] and **no receipt**. `O` (the owner) does not appear in -/// this signature at all — it is never borrowed across the WAL / object-store -/// await, so the owner stays free while durability runs. +/// **Phase 1 — async persistence, NO owner borrow.** Form the crash-durable +/// [`DurableWitness`] and append it CO-LOCATED with the cast's payload in one +/// generation; on success return a [`DurableReceipt`] (which merely references +/// that durable material), on failure a [`PersistError::Write`] and **no +/// receipt**. `O` (the owner) does not appear in this signature at all — it is +/// never borrowed across the WAL / object-store await, so the owner stays free +/// while durability runs. pub async fn persist_cast( sink: &W, cast: PersistCast, ) -> Result { + let witness = DurableWitness { + owner: cast.owner, + cast_id: cast.cast_id, + cycle: cast.cycle, + paired_move: cast.paired_move, + }; let coordinate = sink - .append(cast.owner, &cast.payload) + .append(&witness, &cast.payload) .await .map_err(PersistError::Write)?; Ok(DurableReceipt { owner: cast.owner, + cast_id: cast.cast_id, paired_move: cast.paired_move, coordinate, }) @@ -171,14 +281,66 @@ pub fn apply_durable_step( }); } match receipt.paired_move { - Some(mv) => owner - .try_advance_phase(mv.to) - .map(Some) - .map_err(PersistError::Illegal), + Some(mv) => { + // The move's `from` must match the owner's current phase: on the + // synchronous path a mismatch is a stale / out-of-cast-order apply + // and is surfaced loudly (a double apply would otherwise silently + // re-run through `try_advance_phase`). Enforces cast order. + if mv.from != owner.phase() { + return Err(PersistError::StalePhase { + owner_phase: owner.phase(), + move_from: mv.from, + }); + } + owner + .try_advance_phase(mv.to) + .map(Some) + .map_err(PersistError::Illegal) + } None => Ok(None), } } +/// **Crash recovery.** Given the durable witnesses read back via +/// [`DurableWrite::scan_witnesses`] (a globally-interleaved stream from every +/// owner) and an `owner` reconstructed at its durable phase, re-apply the owner's +/// PENDING paired moves in cast order and return them. +/// +/// The read is `temporal` layer-1: [`local_trajectory_of`] deinterlaces the +/// global stream down to this owner's own chain, in `cast_id` order — the exact +/// replay order. Each move is applied only when its `from` matches the owner's +/// current phase; a move whose `from` no longer matches is **already reflected in +/// the recovered durable SoA state** and is SKIPPED (idempotent — re-running +/// recovery after catching up applies nothing). A move that matches `from` but is +/// not a legal Rubicon edge is a genuine corruption and is surfaced +/// ([`PersistError::Illegal`]). +/// +/// This is why the paired move MUST be co-located in the durable generation: the +/// witnesses are the only reason a KanbanStep survives a crash between the WAL +/// append and [`apply_durable_step`]. No witnesses ⇒ nothing to replay ⇒ the step +/// is lost (the gap this reshape closes). +pub fn recover_and_apply( + owner: &mut O, + witnesses: &[DurableWitness], +) -> Result, PersistError> { + let chain = local_trajectory_of(witnesses, owner.mailbox_id()); + let mut applied = Vec::new(); + for w in chain { + let Some(mv) = w.paired_move else { continue }; + if mv.from != owner.phase() { + // Stale: already reflected in the recovered durable state. Skip + // (recovery is idempotent), do not surface — unlike the synchronous + // single-receipt path, a stale move here is expected, not an error. + continue; + } + let step = owner + .try_advance_phase(mv.to) + .map_err(PersistError::Illegal)?; + applied.push(step); + } + Ok(applied) +} + #[cfg(test)] mod tests { use super::*; @@ -236,37 +398,50 @@ mod tests { } } - /// A `DurableWrite` whose success/failure and call-count are observable. - /// `&self` + interior-mutable counter, mirroring the real sink's shared shape. + /// A `DurableWrite` whose success/failure and call-count are observable, and + /// which RECORDS every witness it lands so [`scan_witnesses`] can replay them + /// — this is what makes the crash-recovery falsifier real: the durable + /// generation (here, `landed`) outlives the in-memory receipts. `&self` + + /// interior mutability, mirroring the real sink's shared shape. struct FakeSink { succeed: bool, calls: std::cell::Cell, + landed: std::cell::RefCell>, } impl FakeSink { fn new(succeed: bool) -> Self { Self { succeed, calls: std::cell::Cell::new(0), + landed: std::cell::RefCell::new(Vec::new()), } } } impl DurableWrite for FakeSink { async fn append( &self, - _owner: MId, + witness: &DurableWitness, _payload: &[u8], ) -> Result { self.calls.set(self.calls.get() + 1); if self.succeed { + // The witness lands CO-LOCATED and DURABLE — it survives even if + // every in-memory receipt is dropped (the crash the reshape guards). + self.landed.borrow_mut().push(witness.clone()); Ok(DurableCoordinate { shard: 0xABCD, writer_epoch: 1, - wal_entry_position: 7, + wal_entry_position: self.landed.borrow().len() as u64, }) } else { + // The negative half: a fenced WAL lands NOTHING — no witness, so + // nothing to replay, so no move and no step. Err(WriteFailed("wal fenced".into())) } } + async fn scan_witnesses(&self) -> Result, WriteFailed> { + Ok(self.landed.borrow().clone()) + } } fn owner(phase: KanbanColumn) -> FakeOwner { @@ -277,11 +452,16 @@ mod tests { } } fn cast(paired_to: Option) -> PersistCast { + cast_from(KanbanColumn::Planning, paired_to) + } + fn cast_from(from: KanbanColumn, paired_to: Option) -> PersistCast { PersistCast { owner: 42, + cast_id: CastId(0), + cycle: 5, paired_move: paired_to.map(|to| KanbanMove { mailbox: 42, - from: KanbanColumn::Planning, + from, to, witness_chain_position: 0, exec: ExecTarget::Elixir, @@ -290,11 +470,15 @@ mod tests { } } fn receipt_for(paired_to: Option) -> DurableReceipt { + receipt_from(KanbanColumn::Planning, paired_to) + } + fn receipt_from(from: KanbanColumn, paired_to: Option) -> DurableReceipt { DurableReceipt { owner: 42, + cast_id: CastId(0), paired_move: paired_to.map(|to| KanbanMove { mailbox: 42, - from: KanbanColumn::Planning, + from, to, witness_chain_position: 0, exec: ExecTarget::Elixir, @@ -321,10 +505,32 @@ mod tests { Some(KanbanColumn::CognitiveWork) ); assert_eq!( - r.coordinate.wal_entry_position, 7, - "the durable coordinate rides" + r.coordinate.wal_entry_position, 1, + "the durable coordinate rides (first witness landed)" ); + assert_eq!(r.cast_id, CastId(0), "the receipt mirrors the cast id"); assert_eq!(sink.calls.get(), 1, "the write was attempted once"); + // The witness LANDED durably (co-located) — not only in the receipt. + let scanned = sink.scan_witnesses().await.expect("scan"); + assert_eq!(scanned.len(), 1, "one witness durably co-located"); + assert_eq!( + scanned[0].paired_move.map(|m| m.to), + Some(KanbanColumn::CognitiveWork), + "the paired move is in the DURABLE witness, not only the in-memory receipt", + ); + } + + #[tokio::test] + async fn a_fenced_write_lands_no_witness_so_nothing_can_be_replayed() { + // Negative half, at the durable layer: a failed append records NO witness, + // so a subsequent crash-recovery scan finds nothing to replay ⇒ no move. + let sink = FakeSink::new(false); + let _ = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; + assert_eq!( + sink.scan_witnesses().await.expect("scan").len(), + 0, + "a fenced WAL leaves no durable witness — no move, no step", + ); } #[tokio::test] @@ -364,9 +570,12 @@ mod tests { "precondition: generic successor is Commit", ); let mut o = owner(KanbanColumn::Evaluation); - let step = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::Prune))) - .expect("legal") - .expect("paired veto"); + let step = apply_durable_step( + &mut o, + receipt_from(KanbanColumn::Evaluation, Some(KanbanColumn::Prune)), + ) + .expect("legal") + .expect("paired veto"); assert_eq!(step.to, KanbanColumn::Prune, "the paired veto, not Commit"); assert_eq!(o.phase(), KanbanColumn::Prune); assert_ne!( @@ -416,4 +625,148 @@ mod tests { "foreign owner untouched" ); } + + // ── Crash recovery: the co-located witness replays after the receipt is lost ─ + + /// Build a `PersistCast` for owner `owner` with an explicit cast id and move. + fn cast_of(owner: MId, cast_id: u64, from: KanbanColumn, to: KanbanColumn) -> PersistCast { + PersistCast { + owner, + cast_id: CastId(cast_id), + cycle: 0, + paired_move: Some(KanbanMove { + mailbox: owner, + from, + to, + witness_chain_position: cast_id as u32, + exec: ExecTarget::Elixir, + }), + payload: vec![cast_id as u8], + } + } + + /// THE crash falsifier (operator): WAL append succeeds, then the process + /// dies BEFORE the sync step — every in-memory [`DurableReceipt`] is dropped. + /// On restart the owner is reconstructed at its durable phase and + /// [`recover_and_apply`] replays the PAIRED move from the co-located witness. + /// Without the reshape the move lived only in the dropped receipt and the + /// KanbanStep would be lost; here it is reconstructed and applied. + #[tokio::test] + async fn a_crash_after_durable_write_replays_the_move_from_the_witness() { + let sink = FakeSink::new(true); + // The write lands durably (witness co-located). + let receipt = persist_cast( + &sink, + cast_of(42, 0, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + ) + .await + .expect("write landed"); + // ── CRASH ── drop the receipt without ever calling apply_durable_step. + drop(receipt); + + // Restart: reconstruct the owner at its DURABLE phase (Planning — the + // step never applied) and recover from what durable storage holds. + let scanned = sink.scan_witnesses().await.expect("scan"); + let mut o = owner(KanbanColumn::Planning); + let applied = recover_and_apply(&mut o, &scanned).expect("recovery legal"); + assert_eq!( + applied.iter().map(|m| m.to).collect::>(), + vec![KanbanColumn::CognitiveWork], + "the pending move was reconstructed from the durable witness and applied", + ); + assert_eq!( + o.phase(), + KanbanColumn::CognitiveWork, + "the step fired on recovery" + ); + + // Idempotent: recovery re-run after catching up applies nothing (the move + // is now reflected in the recovered state — stale, skipped, not errored). + let again = recover_and_apply(&mut o, &scanned).expect("idempotent"); + assert!(again.is_empty(), "second recovery is a no-op"); + assert_eq!(o.phase(), KanbanColumn::CognitiveWork); + } + + /// FALSIFIER (operator): one owner's durable batch replays its moves in + /// CAST order, and the globally-interleaved OTHER owner's witnesses are + /// deinterlaced away (temporal layer-1). The durable log interleaves owner + /// 42 and owner 99; recovery of 42 applies only 42's chain, in cast order. + #[tokio::test] + async fn recovery_replays_one_owners_batch_in_cast_order_ignoring_interleaved_owners() { + let sink = FakeSink::new(true); + // Global durable-log order interleaves two owners: + // c0: 42 Planning→CognitiveWork + // c1: 99 Planning→CognitiveWork (other owner, interleaved) + // c2: 42 CognitiveWork→Evaluation + for c in [ + cast_of(42, 0, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + cast_of(99, 1, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + cast_of(42, 2, KanbanColumn::CognitiveWork, KanbanColumn::Evaluation), + ] { + persist_cast(&sink, c).await.expect("landed"); + } + let scanned = sink.scan_witnesses().await.expect("scan"); + assert_eq!(scanned.len(), 3, "all three witnesses are durable"); + + let mut o = owner(KanbanColumn::Planning); // owner 42 at its durable phase + let applied = recover_and_apply(&mut o, &scanned).expect("legal"); + assert_eq!( + applied.iter().map(|m| m.to).collect::>(), + vec![KanbanColumn::CognitiveWork, KanbanColumn::Evaluation], + "owner 42's OWN chain, in cast order — 99's interleaved cast deinterlaced away", + ); + assert_eq!( + o.phase(), + KanbanColumn::Evaluation, + "advanced two local steps" + ); + } + + /// FALSIFIER 5 (operator): the durable proof is the coordinate, NOT a + /// `LanceVersion`. A witness is queryable/replayable the instant it lands — + /// before any base manifest version attaches — so recovery identifies the + /// latest LOCAL state from the co-located witness cast order, not a dataset + /// version. Here no dataset version exists at all, yet recovery is exact. + #[tokio::test] + async fn recovery_uses_cast_order_not_a_dataset_version() { + let sink = FakeSink::new(true); + let receipt = persist_cast( + &sink, + cast_of(42, 0, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + ) + .await + .expect("landed"); + // The durability proof carries no dataset version — only a WAL/LSM + // coordinate. (The type has no `DatasetVersion` field to read.) + assert!( + receipt.coordinate.wal_entry_position > 0, + "durable via the WAL coordinate, before any manifest version", + ); + let scanned = sink.scan_witnesses().await.expect("scan"); + let mut o = owner(KanbanColumn::Planning); + let applied = recover_and_apply(&mut o, &scanned).expect("legal"); + assert_eq!( + applied.len(), + 1, + "latest local state found from cast order alone" + ); + } + + /// The synchronous single-receipt path surfaces a stale/out-of-order move + /// LOUDLY ([`PersistError::StalePhase`]) — the counterpart to recovery's + /// silent stale-skip. A receipt whose move is `Planning→…` applied to an + /// owner already at `CognitiveWork` is refused, owner untouched. + #[test] + fn a_stale_move_on_the_sync_path_is_surfaced_not_silently_reapplied() { + let mut o = owner(KanbanColumn::CognitiveWork); + let r = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::CognitiveWork))); + assert_eq!( + r, + Err(PersistError::StalePhase { + owner_phase: KanbanColumn::CognitiveWork, + move_from: KanbanColumn::Planning, + }), + ); + assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "owner untouched"); + } } diff --git a/crates/lance-graph-planner/src/temporal.rs b/crates/lance-graph-planner/src/temporal.rs index 65874d31..e384d0b3 100644 --- a/crates/lance-graph-planner/src/temporal.rs +++ b/crates/lance-graph-planner/src/temporal.rs @@ -43,6 +43,30 @@ //! so the single-server body can ignore them and the cross-server policy (peer- //! Raft / cluster bus) wakes them up with **no breaking signature change** — //! avoiding the `emitted_at_millis: u64` (decision #4) non-`Option` trap. +//! +//! ## Two temporal LAYERS (operator-ruled — do not conflate) +//! +//! `temporal.rs` deinterlaces on **two distinct axes**, and both are needed +//! before a durable kanban step can be trusted after a crash: +//! +//! - **Layer 1 — CAUSAL deinterlacing** ([`local_trajectories`], +//! [`LocalCausalRow`]): the global durable log interleaves *every* owner's +//! writes — `A@s0, C@s0, B@s0, A@s1`. Splitting it back into per-owner LOCAL +//! chains — owner A's own `[A@s0, A@s1]`, with B's and C's rows *removed* from +//! A's timeline — is what makes crash-replay in cast order possible. Answers +//! *"what is owner A's own trajectory?"* +//! - **Layer 2 — EPISTEMIC projection** ([`classify`], [`deinterlace`]): given a +//! trajectory, which rows may a reader at a given rung/horizon dispatch on +//! (`Contemporary` / `Anachronistic` / `Spoiler` / `Unknowable`). Answers +//! *"what may this reader know?"* +//! +//! They compose: layer 1 reconstructs an owner's local causal chain from the +//! interleaved log; layer 2 filters that chain by the reader's horizon. A crash +//! recovery reads the durable witnesses, runs layer 1 to find each owner's +//! pending tail, and re-applies in cast order (`persist_sink::recover_and_apply`). + +use lance_graph_contract::collapse_gate::MailboxId; +use std::collections::BTreeMap; /// A Lance dataset version — the storage frame's clock tick. pub type LanceVersion = u64; @@ -351,6 +375,69 @@ where out } +// ───────────────────────────────────────────────────────────────────────────── +// LAYER 1 — CAUSAL DEINTERLACING (global interleaved durable log → per-owner +// local trajectories). Distinct from the layer-2 epistemic projection above: +// layer 2 asks *what a reader may see*; layer 1 asks *what one owner actually +// did, in its own order*. The interleaved global log +// A@s0, C@s0, B@s0, A@s1 +// yields owner A's LOCAL chain [A@s0, A@s1] — B's and C's rows are REMOVED from +// A's timeline (deinterlaced away), not merely reordered. This is the read +// crash-recovery runs to find each owner's pending durable tail. +// ───────────────────────────────────────────────────────────────────────────── + +/// A row in the global durable log carrying its owner-local causal coordinates — +/// WHO wrote it and WHERE in that owner's own cast sequence. +/// +/// The layer-1 counterpart of [`DeinterlaceRow`]: where `DeinterlaceRow` exposes +/// the *epistemic* clocks (lance version / knowable-from / HLC), this exposes the +/// *causal* coordinates (owner + cast sequence) that split the interleaved stream +/// into per-owner local chains. A durable persistence witness +/// (`persist_sink::DurableWitness`) is the production implementor. +pub trait LocalCausalRow { + /// The mailbox that owns this row — the deinterlace grouping key. + fn owner(&self) -> MailboxId; + /// The owner-local monotonic cast sequence (e.g. `CastId.0`) — the ordering + /// key WITHIN one owner's trajectory. Cross-owner values are never compared; + /// only rows sharing an `owner()` are ordered against each other. + fn cast_seq(&self) -> u64; +} + +/// Layer-1 causal deinterlacing: split a globally-interleaved durable stream into +/// per-owner LOCAL trajectories, each ordered by the owner's own `cast_seq`. +/// +/// The global order `A@s0, C@s0, B@s0, A@s1` yields +/// `{A: [A@s0, A@s1], B: [B@s0], C: [C@s0]}` — every owner's interleaved +/// neighbours are removed from its timeline, and each timeline is put back into +/// the owner's own cast order (the crash-replay order). Returns a `BTreeMap` so +/// owners iterate deterministically. +#[must_use] +pub fn local_trajectories(global: &[R]) -> BTreeMap> { + let mut by_owner: BTreeMap> = BTreeMap::new(); + for row in global { + by_owner.entry(row.owner()).or_default().push(row.clone()); + } + for rows in by_owner.values_mut() { + rows.sort_by_key(LocalCausalRow::cast_seq); + } + by_owner +} + +/// Owner `owner`'s local trajectory alone — the crash-recovery read for ONE +/// owner. Filters the interleaved global stream to that owner's rows and orders +/// them by `cast_seq`. Equivalent to `local_trajectories(global).remove(&owner)` +/// but without materialising the other owners' chains. +#[must_use] +pub fn local_trajectory_of(global: &[R], owner: MailboxId) -> Vec { + let mut chain: Vec = global + .iter() + .filter(|r| r.owner() == owner) + .cloned() + .collect(); + chain.sort_by_key(LocalCausalRow::cast_seq); + chain +} + #[cfg(test)] mod tests { use super::*; @@ -403,6 +490,117 @@ mod tests { } } + /// A minimal [`LocalCausalRow`] — owner + owner-local cast sequence, plus a + /// `tag` to make the surviving order observable. + #[derive(Clone, Debug, PartialEq)] + struct CausalRow { + owner: MailboxId, + seq: u64, + tag: &'static str, + } + impl LocalCausalRow for CausalRow { + fn owner(&self) -> MailboxId { + self.owner + } + fn cast_seq(&self) -> u64 { + self.seq + } + } + + /// LAYER-1 FALSIFIER (operator): the interleaved global log + /// `A@s0, C@s0, B@s0, A@s1` must yield owner A's LOCAL chain `[A@s0, A@s1]` — + /// C's and B's interleaved rows REMOVED from A's timeline, not reordered into + /// it. Vacuity guard: A's chain must be strictly shorter than the global log + /// (the other owners' rows were actually dropped, not merely sorted). + #[test] + fn layer1_deinterlaces_interleaved_global_log_into_owner_local_chain() { + const A: MailboxId = 40; + const B: MailboxId = 41; + const C: MailboxId = 42; + // Global durable-log order, exactly the operator's example. + let global = vec![ + CausalRow { + owner: A, + seq: 0, + tag: "A@s0", + }, + CausalRow { + owner: C, + seq: 0, + tag: "C@s0", + }, + CausalRow { + owner: B, + seq: 0, + tag: "B@s0", + }, + CausalRow { + owner: A, + seq: 1, + tag: "A@s1", + }, + ]; + + let a_chain = local_trajectory_of(&global, A); + assert_eq!( + a_chain.iter().map(|r| r.tag).collect::>(), + vec!["A@s0", "A@s1"], + "owner A's local trajectory is its own two casts, deinterlaced", + ); + assert!( + a_chain.len() < global.len(), + "the other owners' interleaved rows were REMOVED from A's timeline, not reordered", + ); + // The same via the all-owners split. + let all = local_trajectories(&global); + assert_eq!(all.len(), 3, "three distinct owners"); + assert_eq!( + all[&A].iter().map(|r| r.tag).collect::>(), + vec!["A@s0", "A@s1"] + ); + assert_eq!( + all[&B].iter().map(|r| r.tag).collect::>(), + vec!["B@s0"] + ); + assert_eq!( + all[&C].iter().map(|r| r.tag).collect::>(), + vec!["C@s0"] + ); + } + + /// LAYER-1 FALSIFIER (operator): within ONE owner, replay order is the cast + /// sequence, even when the durable log stored the casts out of order (a WAL + /// that interleaved a later batch ahead of an earlier one). Falsifiable: the + /// global log is deliberately seq-descending, so a no-op (identity) pass + /// would produce the wrong order. + #[test] + fn layer1_orders_one_owners_chain_by_cast_seq_not_log_order() { + const A: MailboxId = 7; + let global = vec![ + CausalRow { + owner: A, + seq: 2, + tag: "third", + }, + CausalRow { + owner: A, + seq: 0, + tag: "first", + }, + CausalRow { + owner: A, + seq: 1, + tag: "second", + }, + ]; + let chain = local_trajectory_of(&global, A); + assert_eq!( + chain.iter().map(|r| r.tag).collect::>(), + vec!["first", "second", "third"], + "replay follows cast_seq, not the (scrambled) durable-log order", + ); + } + #[test] fn for_rung_policy() { assert_eq!(EpistemicMode::for_rung(0), EpistemicMode::Strict); From c7424cd986ad8d3ace41adc4f80f53c9e71c985a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 23:31:15 +0000 Subject: [PATCH 4/9] docs(board): D-MBX-A6-P3d durable-witness reshape + temporal layer-1 Board hygiene for the persist_sink / temporal changes (same logical unit): - EPIPHANIES: E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1 (the crash gap, the co-location rule, temporal's missing layer-1, 5 falsifiers) - STATUS_BOARD: D-MBX-A6-P3d row (persistence ordering core + reshape + layer-1) - LATEST_STATE: persist_sink + temporal layer-1 contract-inventory entries Concrete LanceShardSink remains deferred per operator ruling. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 16 ++++++++++++++++ .claude/board/LATEST_STATE.md | 7 +++++++ .claude/board/STATUS_BOARD.md | 1 + 3 files changed, 24 insertions(+) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 3468b802..37cb8627 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,19 @@ +## 2026-08-01 — E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1 — the persistence sink's crash gap, and temporal.rs's missing layer-1 + +**Status:** FINDING (operator-ruled 2026-08-01). **Confidence:** High — closed with five falsifiable crash/recovery probes (`lance_graph_planner::persist_sink` + `temporal`); 343 planner lib tests green, clippy+fmt clean; builds NO concrete sink. + +**The gap the split-phase left open.** At the prior two-clock-domain shape, `persist_cast` appended only the PAYLOAD; the paired `KanbanMove` lived solely in the in-memory `DurableReceipt`. Sequence: WAL append lands (durable) → process dies before `apply_durable_step` → the receipt evaporates → the paired move is lost → the KanbanStep never fires — **even though the durable SoA state moved**. Silent: storage advanced, lifecycle did not. An in-memory receipt is not a durable record; a step that lives only in it is a step that a crash can delete. + +**The rule (operator).** The paired transition witness must be **durably co-located with the SoA state in the same persistence generation**. The in-memory receipt may REFERENCE that durable material (via its `DurableCoordinate`) but must never be its only copy. NOT via a separate ack/confirmation ledger (`E-ACK-ELIMINATED-1` stands) — the witness rides IN the generation and is read back with the state. So `DurableWrite::append(&DurableWitness, &payload)` lands both atomically; `scan_witnesses` reads them back; `recover_and_apply` replays the pending tail. + +**temporal.rs is TWO layers, and layer-1 was missing.** Layer-2 (epistemic projection — `classify`/`deinterlace`: contemporary/anachronistic/spoiler/unknowable) was well-developed. Layer-1 (**causal** deinterlacing) did not exist: the global durable log interleaves every owner's writes (`A@s0, C@s0, B@s0, A@s1`), and reconstructing owner A's OWN local chain `[A@s0, A@s1]` — with the interleaved owners *removed* from A's timeline — is what makes crash-replay in cast order possible. New `LocalCausalRow` + `local_trajectories`/`local_trajectory_of`. The two layers compose: layer-1 rebuilds the chain, layer-2 filters it by the reader's horizon. `DurableWitness` implements `LocalCausalRow`. + +**Durability proof is the coordinate, not a `LanceVersion`.** A MemWAL write is queryable/durable BEFORE any base manifest version attaches, so recovery identifies the latest LOCAL state from the co-located witness's cast order — not from a dataset version that does not yet exist (falsifier 5). This is why `DurableCoordinate` (shard/epoch/wal-position), never `DatasetVersion`, is what `append` returns. + +**Five integration falsifiers (all green).** (1) layer-1 deinterlaces the interleaved global log into an owner-local chain, with a vacuity guard that the other owners' rows are *dropped* not reordered; (2) one owner's batch replays in cast order over a deliberately seq-descending log; (3) WAL succeeds + crash (receipt dropped) → recovery reconstructs+applies the move from the durable witness, idempotent on re-run; (4) WAL fails → no witness lands → no move, no step; (5) WAL-visible-before-manifest → recovery uses cast order, no dataset version exists. Plus a synchronous-path `StalePhase` guard (a stale/out-of-order apply is surfaced loudly on `apply_durable_step`, while recovery silently skips a stale move as already-reflected). + +**Still deferred (operator ruling — do NOT build yet).** The concrete `LanceShardSink` (`ShardWriter::put`, `enable_memtable + durable_write`) comes AFTER, gated on these falsifiers; the durable local causal chain lands first. No confirmation ledger, replay queue, per-thought ack, custom WAL, or ractor hot-path callback. Extends `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1`. + ## 2026-08-01 — E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1 — the D-MBX-A6 move is pre-write intent; the KanbanStep is post-write; the persistence sink wires Lance 7's existing MemWAL and invents nothing **Status:** FINDING (operator-ruled 2026-08-01). **Confidence:** High — the pre-write half is shipped with 5 falsifiable probes (`lance_graph_planner::owner_adapter`); the persistence half's API is verified against real `lance-7.0.0` source. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index d632f6de..27ef7144 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,10 @@ +## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3d persistence-sink durable-witness reshape + temporal layer-1 + +### Current Contract Inventory — new/changed modules (lance-graph-planner) +- `lance_graph_planner::persist_sink` — the POST-write half, two clock domains, **crash-durable**. `DurableWitness{owner, cast_id, cycle, paired_move}` is CO-LOCATED with the SoA payload in one persistence generation via `DurableWrite::append(&witness, &payload)`; the in-memory `DurableReceipt` merely REFERENCES it (via `DurableCoordinate`), never the only copy of the move. `DurableWrite::scan_witnesses` = replay seam; `recover_and_apply(owner, witnesses)` = crash recovery (temporal layer-1 → per-owner pending tail, replayed in cast order, idempotent stale-skip). `apply_durable_step` gains a `from==phase` guard (`PersistError::StalePhase`). Async `persist_cast` (no owner borrow) / sync `apply_durable_step` (no await) split preserved. Durability proof = `DurableCoordinate` (shard/epoch/wal-position), NEVER `LanceVersion`. **No concrete `LanceShardSink` built** (deferred per operator, gated on the crash falsifiers). +- `lance_graph_planner::temporal` — **layer-1 CAUSAL deinterlacing added** (the missing half): `LocalCausalRow{owner, cast_seq}` + `local_trajectories`/`local_trajectory_of` split a globally-interleaved durable log into per-owner LOCAL chains ordered by `cast_seq` (`A@s0,C@s0,B@s0,A@s1` → A's `[A@s0,A@s1]`; interleaved owners removed). Composes with the pre-existing layer-2 epistemic projection (`classify`/`deinterlace`). `DurableWitness` implements `LocalCausalRow`. +- Gates: 5/5 operator crash/recovery falsifiers green + layer-1 falsifiers; 343 planner lib tests, `cargo clippy -p lance-graph-planner` + `cargo fmt -p` clean. `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1`. + ## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3c owner-consume adapter (rebased onto main dcd9cc9) ### Current Contract Inventory — new module (lance-graph-planner) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 823de65e..602e337c 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -963,6 +963,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3a | StyleStrategy: thinking-style -> cluster -> mechanism -> recipe_kernels Tactic selection (planning substrate; carries tau JIT addr) | lance-graph-planner | 130 | LOW | **In PR** | #439; first cut of A6-P3 consumer wiring; planner now consumes contract recipes/styles; deferred: i4-32D decode, Outcome->Candidate, tau->JIT, membrane commit | | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | +| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses` replay seam; `recover_and_apply` replays the pending tail in cast order (idempotent, stale-skip); `StalePhase` from-guard on the sync path. `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate`, never `LanceVersion`. | lance-graph-planner | 130 | LOW | **In progress** | 5 operator falsifiers green (deinterlace+vacuity, cast-order replay, crash→reconstruct+apply, WAL-fail→no-step, WAL-before-manifest); 343 planner lib tests, clippy+fmt clean; builds NO concrete `LanceShardSink` (deferred per operator — gated on these falsifiers); `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | From 08fa9f351115346742428fa4a5fa7298639024d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 23:47:57 +0000 Subject: [PATCH 5/9] fix(planner): durable-position replay order + watermark idempotence (PR #878 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three confirmed correctness fixes from Bugbot/Codex/CodeRabbit review, all in the crash-recovery path the operator mandated: - **cast_id resets across restarts (Bugbot High).** BatchWriter::next_id resets to 0 on restart, so witnesses from different writer lifetimes collide on cast_id — ordering replay by it is unstable. Replay now orders by the durable DurableCoordinate::log_order (WAL entry position, monotonic across restarts, never reset). scan_witnesses returns LandedWitness{coordinate, witness}, which is the LocalCausalRow implementor; cast_id survives as provenance only. - **phase equality is unsound idempotence on a cyclic lifecycle (Codex + CodeRabbit Critical).** Planning→CognitiveWork→Evaluation→Plan→Planning is a real lap; after it the owner is back at Planning, so a phase-only "already applied" check replays the whole lap. recover_and_apply now takes a durable watermark (applied_through): skip ≤ watermark; above it a non-matching `from` is genuine corruption (StalePhase), not a benign skip. Returns Recovered{applied, watermark} to persist WITH the SoA phase. New falsifier proves cyclic idempotence WITH the watermark and, as a negative control, reproduces the double-lap WITHOUT it (watermark is load-bearing). - **cross-owner move could become durable (CodeRabbit Major).** persist_cast now rejects paired_move.mailbox != owner before the append; apply_durable_step adds the same defense-in-depth check. Also: scan_witnesses(from) is bounded (tail read); WriteFailed/PersistError impl Display+Error (no snafu dep added); FakeSink is Sync (Mutex+Atomic) with a multi-thread concurrent-drain test; a durable-no-step recovery case; the layer-1 cast_seq doc states the monotonic-across-restarts precondition. StalePhase on the sync path documented safe-to-drop (the durable witness replays). 348 planner lib tests pass; clippy + fmt clean. No Cargo.lock change. Still builds NO concrete LanceShardSink (operator-deferred). Board entries (EPIPHANIES Correction, LATEST_STATE, STATUS_BOARD) updated to the corrected design. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 4 +- .claude/board/LATEST_STATE.md | 6 +- .claude/board/STATUS_BOARD.md | 2 +- .../lance-graph-planner/src/persist_sink.rs | 838 +++++++++++++----- crates/lance-graph-planner/src/temporal.rs | 15 +- 5 files changed, 651 insertions(+), 214 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 37cb8627..c7d962ff 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,6 +1,6 @@ ## 2026-08-01 — E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1 — the persistence sink's crash gap, and temporal.rs's missing layer-1 -**Status:** FINDING (operator-ruled 2026-08-01). **Confidence:** High — closed with five falsifiable crash/recovery probes (`lance_graph_planner::persist_sink` + `temporal`); 343 planner lib tests green, clippy+fmt clean; builds NO concrete sink. +**Status:** FINDING (operator-ruled 2026-08-01; review-hardened — see the Correction below). **Confidence:** High — closed with falsifiable crash/recovery probes (`lance_graph_planner::persist_sink` + `temporal`); 348 planner lib tests green, clippy+fmt clean; builds NO concrete sink. **The gap the split-phase left open.** At the prior two-clock-domain shape, `persist_cast` appended only the PAYLOAD; the paired `KanbanMove` lived solely in the in-memory `DurableReceipt`. Sequence: WAL append lands (durable) → process dies before `apply_durable_step` → the receipt evaporates → the paired move is lost → the KanbanStep never fires — **even though the durable SoA state moved**. Silent: storage advanced, lifecycle did not. An in-memory receipt is not a durable record; a step that lives only in it is a step that a crash can delete. @@ -14,6 +14,8 @@ **Still deferred (operator ruling — do NOT build yet).** The concrete `LanceShardSink` (`ShardWriter::put`, `enable_memtable + durable_write`) comes AFTER, gated on these falsifiers; the durable local causal chain lands first. No confirmation ledger, replay queue, per-thought ack, custom WAL, or ractor hot-path callback. Extends `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1`. +**Correction (PR #878 review, same PR — Bugbot High + Codex/CodeRabbit Critical).** The first cut ordered replay by `cast_id` and inferred "already applied" from phase equality. Both are unsound and were fixed within the PR: (a) **`CastId` resets to 0 on every writer restart** (`BatchWriter::next_id`), so cross-lifetime witnesses collide — replay now orders by the durable **`DurableCoordinate::log_order`** (WAL entry position, monotonic across restarts, never reset); `cast_id` survives only as provenance. `scan_witnesses` returns `LandedWitness{coordinate, witness}` and is the `LocalCausalRow` implementor. (b) **Phase equality is not a sound idempotence key on a CYCLIC lifecycle** (`Planning→CognitiveWork→Evaluation→Plan→Planning`): after a lap the owner is back at `Planning`, so a phase-only check replays the whole lap. `recover_and_apply` now takes a durable **watermark** (`applied_through`) — skip ≤ watermark, and above it a non-matching `from` is genuine corruption (`StalePhase`), not a benign skip; it returns the new watermark to persist WITH the SoA phase. Also: `paired_move.mailbox == owner` is validated at persist time (no cross-owner move becomes durable); `scan_witnesses(from)` is bounded; `WriteFailed`/`PersistError` impl `Display`+`Error`. New falsifiers: cyclic-idempotence with a negative control proving the watermark is load-bearing, and cast_id-collision ordering-by-durable-position. The layer-1 `cast_seq` doc now states the monotonic-across-restarts precondition. + ## 2026-08-01 — E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1 — the D-MBX-A6 move is pre-write intent; the KanbanStep is post-write; the persistence sink wires Lance 7's existing MemWAL and invents nothing **Status:** FINDING (operator-ruled 2026-08-01). **Confidence:** High — the pre-write half is shipped with 5 falsifiable probes (`lance_graph_planner::owner_adapter`); the persistence half's API is verified against real `lance-7.0.0` source. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 27ef7144..f671d09f 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,9 +1,9 @@ ## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3d persistence-sink durable-witness reshape + temporal layer-1 ### Current Contract Inventory — new/changed modules (lance-graph-planner) -- `lance_graph_planner::persist_sink` — the POST-write half, two clock domains, **crash-durable**. `DurableWitness{owner, cast_id, cycle, paired_move}` is CO-LOCATED with the SoA payload in one persistence generation via `DurableWrite::append(&witness, &payload)`; the in-memory `DurableReceipt` merely REFERENCES it (via `DurableCoordinate`), never the only copy of the move. `DurableWrite::scan_witnesses` = replay seam; `recover_and_apply(owner, witnesses)` = crash recovery (temporal layer-1 → per-owner pending tail, replayed in cast order, idempotent stale-skip). `apply_durable_step` gains a `from==phase` guard (`PersistError::StalePhase`). Async `persist_cast` (no owner borrow) / sync `apply_durable_step` (no await) split preserved. Durability proof = `DurableCoordinate` (shard/epoch/wal-position), NEVER `LanceVersion`. **No concrete `LanceShardSink` built** (deferred per operator, gated on the crash falsifiers). -- `lance_graph_planner::temporal` — **layer-1 CAUSAL deinterlacing added** (the missing half): `LocalCausalRow{owner, cast_seq}` + `local_trajectories`/`local_trajectory_of` split a globally-interleaved durable log into per-owner LOCAL chains ordered by `cast_seq` (`A@s0,C@s0,B@s0,A@s1` → A's `[A@s0,A@s1]`; interleaved owners removed). Composes with the pre-existing layer-2 epistemic projection (`classify`/`deinterlace`). `DurableWitness` implements `LocalCausalRow`. -- Gates: 5/5 operator crash/recovery falsifiers green + layer-1 falsifiers; 343 planner lib tests, `cargo clippy -p lance-graph-planner` + `cargo fmt -p` clean. `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1`. +- `lance_graph_planner::persist_sink` — the POST-write half, two clock domains, **crash-durable**. `DurableWitness{owner, cast_id, cycle, paired_move}` is CO-LOCATED with the SoA payload in one persistence generation via `DurableWrite::append(&witness, &payload)`; the in-memory `DurableReceipt` merely REFERENCES it (via `DurableCoordinate`), never the only copy of the move. **Replay order = the durable `DurableCoordinate::log_order` (WAL position, monotonic across restarts), NOT `cast_id`** (`BatchWriter`'s counter resets on restart → cross-lifetime collisions; `cast_id` is provenance only). `DurableWrite::scan_witnesses(from)` returns `LandedWitness{coordinate, witness}` (bounded tail read; `LandedWitness` is the `LocalCausalRow` implementor). `recover_and_apply(owner, landed, applied_through)` = crash recovery: temporal layer-1 → per-owner tail in durable order; **idempotence is a durable WATERMARK** (`applied_through`), NOT phase equality (unsound on the cyclic lifecycle — a completed lap returns to `Planning`); above the watermark a non-matching `from` is corruption (`StalePhase`); returns `Recovered{applied, watermark}` to persist with the SoA phase. `persist_cast` validates `paired_move.mailbox == owner` (no cross-owner move becomes durable); `apply_durable_step` keeps the sync `from==phase` guard (a stale drop is safe — the durable witness replays). `WriteFailed`/`PersistError` impl `Display`+`Error`. Async `persist_cast` (no owner borrow) / sync `apply_durable_step` (no await) split preserved. Durability proof = `DurableCoordinate`, NEVER `LanceVersion`. **No concrete `LanceShardSink` built** (deferred per operator, gated on the crash falsifiers). +- `lance_graph_planner::temporal` — **layer-1 CAUSAL deinterlacing added** (the missing half): `LocalCausalRow{owner, cast_seq}` + `local_trajectories`/`local_trajectory_of` split a globally-interleaved durable log into per-owner LOCAL chains ordered by `cast_seq` (`A@s0,C@s0,B@s0,A@s1` → A's `[A@s0,A@s1]`; interleaved owners removed). `cast_seq` doc states the monotonic-across-restarts precondition (a resettable counter is NOT valid; use a durable-log position). Composes with the pre-existing layer-2 epistemic projection (`classify`/`deinterlace`). `LandedWitness` implements `LocalCausalRow`. +- Gates: operator crash/recovery falsifiers green + layer-1 falsifiers + review-driven additions (cyclic-idempotence with negative control; cast_id-collision ordered by durable position; cross-owner reject; concurrent drains; bounded scan); 348 planner lib tests, `cargo clippy -p lance-graph-planner` + `cargo fmt -p` clean. `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1`. ## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3c owner-consume adapter (rebased onto main dcd9cc9) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 602e337c..a7ff8678 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -963,7 +963,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3a | StyleStrategy: thinking-style -> cluster -> mechanism -> recipe_kernels Tactic selection (planning substrate; carries tau JIT addr) | lance-graph-planner | 130 | LOW | **In PR** | #439; first cut of A6-P3 consumer wiring; planner now consumes contract recipes/styles; deferred: i4-32D decode, Outcome->Candidate, tau->JIT, membrane commit | | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | -| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses` replay seam; `recover_and_apply` replays the pending tail in cast order (idempotent, stale-skip); `StalePhase` from-guard on the sync path. `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate`, never `LanceVersion`. | lance-graph-planner | 130 | LOW | **In progress** | 5 operator falsifiers green (deinterlace+vacuity, cast-order replay, crash→reconstruct+apply, WAL-fail→no-step, WAL-before-manifest); 343 planner lib tests, clippy+fmt clean; builds NO concrete `LanceShardSink` (deferred per operator — gated on these falsifiers); `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | +| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate`, never `LanceVersion`. | lance-graph-planner | 160 | LOW | **In PR** | #878; operator falsifiers green + review-hardened (Bugbot High: cast_id-resets→order by durable position; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark w/ negative control; cross-owner reject; concurrent drains; bounded scan); 348 planner lib tests, clippy+fmt clean; builds NO concrete `LanceShardSink` (deferred per operator — gated on these falsifiers); `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs index 2e48bffa..d535d108 100644 --- a/crates/lance-graph-planner/src/persist_sink.rs +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -26,7 +26,9 @@ //! [`DurableReceipt`], and [`apply_durable_step`] cannot be reached without one. //! - **The step is the PAIRED move** (`receipt.paired_move.to`), never a generic //! `next_phases().first()`. The move the thought cast rides in the receipt. -//! - A receipt is applied only to **its own** owner ([`PersistError::OwnerMismatch`]). +//! - A receipt is applied only to **its own** owner ([`PersistError::OwnerMismatch`]), +//! and only if the move itself was minted for that owner (`mv.mailbox == owner` — +//! checked at persist time so a cross-owner move never becomes durable). //! //! ## Crash-durability: the paired move is CO-LOCATED, not in-memory-only //! @@ -43,10 +45,34 @@ //! [`DurableReceipt`] then merely *references* that durable material (via its //! [`DurableCoordinate`]) for the fast in-process apply; it is **not the only //! copy**. On restart, [`recover_and_apply`] reads the witnesses back -//! ([`DurableWrite::scan_witnesses`]), runs `temporal` layer-1 causal -//! deinterlacing to find each owner's pending tail, and re-applies the moves in -//! cast order. This is a read of what durable storage already holds — **not** a -//! separate ack / confirmation ledger (`E-ACK-ELIMINATED-1`). +//! ([`DurableWrite::scan_witnesses`]) and re-applies each owner's pending tail. +//! This is a read of what durable storage already holds — **not** a separate +//! ack / confirmation ledger (`E-ACK-ELIMINATED-1`). +//! +//! ## Replay order + idempotence — the DURABLE coordinate, never `CastId` +//! +//! Two distinct keys, both durable, neither the resettable `CastId` counter: +//! +//! - **Replay ORDER = the durable-log position** ([`DurableCoordinate::log_order`], +//! i.e. `wal_entry_position`). `BatchWriter`'s `CastId` counter **resets to 0 on +//! every restart**, so two witnesses from different writer lifetimes can share a +//! `cast_id` — ordering by it is unstable across crashes. The WAL position is +//! monotonic across the shard's whole life and never resets; `writer_epoch` +//! fences overlapping writers so positions never collide. One owner writes one +//! shard ⇒ a total order over that owner's witnesses, valid across crashes. +//! `cast_id` survives on the witness only as provenance/audit. +//! - **Idempotence = a durable WATERMARK** (`applied_through`), the last durable +//! coordinate whose move is already reflected in the recovered SoA state. +//! **Phase equality is NOT a sound idempotence key**: the Rubicon lifecycle is +//! cyclic (`Planning → CognitiveWork → Evaluation → Plan → Planning`), so after +//! a completed lap the owner is back at `Planning` and a phase-only check would +//! replay the whole lap. [`recover_and_apply`] skips every witness at or below +//! the watermark; above it, the chain must be contiguous (a `from` that does not +//! match the current phase is a genuine gap/corruption, surfaced as +//! [`PersistError::StalePhase`], never silently skipped). **The watermark MUST +//! be persisted with the SoA state** (same generation as the phase it agrees +//! with) or the cyclic ambiguity returns after a second crash — a single +//! per-owner scalar high-water mark, not a per-thought ledger. //! //! ## The `DurableWrite` seam + the durability type //! @@ -57,8 +83,8 @@ //! not `LanceVersion`, is the durability proof: the write is queryable and //! durable *before* any base manifest version attaches (falsifier 5 — //! WAL-visible-before-manifest — proves the latest local state is identified from -//! the co-located witness's cast order, not from a dataset version that does not -//! exist yet). +//! the co-located witness's durable position, not from a dataset version that +//! does not exist yet). //! //! The concrete impl (a lance-having crate) wires lance 7.0.0's OFFICIAL MemWAL — //! preferring the high-level `ShardWriter::put` (`enable_memtable + durable_write`: @@ -66,7 +92,9 @@ //! off-lock) over the raw `WalAppender::append` primitive. The witness and the //! payload are two columns of ONE `RecordBatch` (one generation, atomic), whose //! buffers are independently owned, so "no owner borrow across I/O" holds without -//! any claim of magic zero-copy. +//! any claim of magic zero-copy. [`DurableWrite::scan_witnesses`] takes a `from` +//! lower bound so recovery reads only the tail after the last applied coordinate, +//! never the whole log. //! //! Keeping the seam a trait leaves the ordering core lance-free and `protoc`-free. //! **This module builds NO concrete `LanceShardSink`** — per operator ruling the @@ -84,6 +112,13 @@ use crate::temporal::{local_trajectory_of, LocalCausalRow}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct WriteFailed(pub String); +impl std::fmt::Display for WriteFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "durable write did not land: {}", self.0) + } +} +impl std::error::Error for WriteFailed {} + /// A backend-neutral **durable coordinate** — where the write landed in the /// durable log. This is NOT a base Lance `DatasetVersion` (that arrives later via /// MemTable flush + manifest commit); it is the WAL/LSM coordinate that proves @@ -93,27 +128,43 @@ pub struct WriteFailed(pub String); pub struct DurableCoordinate { /// Opaque shard identity (lance: the shard `Uuid` as `u128`). pub shard: u128, - /// The writer epoch that fenced this append. + /// The writer epoch that fenced this append. A new writer lifetime takes a + /// higher epoch; fencing guarantees two epochs never share a WAL position. pub writer_epoch: u64, - /// The monotonic WAL entry position of this durable append. + /// The monotonic WAL entry position of this durable append. Monotonic across + /// the shard's whole life — it does **not** reset on writer restart. pub wal_entry_position: u64, } +impl DurableCoordinate { + /// The durable-log total-order key for replay + the watermark comparison. + /// + /// `wal_entry_position` is monotonic across the shard's whole life and never + /// resets (unlike `BatchWriter`'s `CastId` counter), and `writer_epoch` fences + /// overlapping writers so positions never collide. One owner writes one shard, + /// so this is a total order over that owner's witnesses, valid across crashes. + #[must_use] + pub fn log_order(&self) -> u64 { + self.wal_entry_position + } +} + /// The durable transition witness — CO-LOCATED with the SoA payload in the same /// persistence generation so it survives a crash (module doc § Crash-durability). /// /// Everything needed to re-apply a pending KanbanStep after a restart lives here: -/// WHO (`owner`), WHICH cast (`cast_id` — the owner-local replay order), the -/// owner-local `cycle`, and the `paired_move` to apply. On recovery, -/// [`recover_and_apply`] reads these back and replays in `cast_id` order. -/// -/// Implements [`LocalCausalRow`] so `temporal` layer-1 can deinterlace a global -/// interleaved witness stream into each owner's local trajectory. +/// WHO (`owner`), the `paired_move` to apply, and the owner-local `cycle`. +/// `cast_id` rides as **provenance/audit only** — it is NOT the replay-order key +/// (it resets across restarts); the durable [`DurableCoordinate`] the sink assigns +/// at append time is, and it is carried by [`LandedWitness`] on the read path. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DurableWitness { - /// The mailbox this write is on behalf of — the deinterlace grouping key. + /// The mailbox this write is on behalf of — the deinterlace grouping key. The + /// `paired_move` (when present) is minted for this same mailbox — checked at + /// persist time, so a cross-owner move never becomes durable. pub owner: MailboxId, - /// The cast this write realises — its `.0` is the owner-local replay order. + /// The cast this write realises — **provenance only** (resets across restarts; + /// never the replay-order key). The durable coordinate is the order. pub cast_id: CastId, /// The owner-local cycle at the time of the cast (audit / trajectory). pub cycle: u32, @@ -122,12 +173,26 @@ pub struct DurableWitness { pub paired_move: Option, } -impl LocalCausalRow for DurableWitness { +/// A [`DurableWitness`] read back from the durable log WITH the durable coordinate +/// the sink assigned at append time — the read-path shape [`scan_witnesses`] +/// returns. The coordinate is what orders replay + drives the idempotence +/// watermark (never the resettable `cast_id`), so it implements [`LocalCausalRow`] +/// keyed on [`DurableCoordinate::log_order`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LandedWitness { + /// Where this witness durably landed — the total-order key across restarts. + pub coordinate: DurableCoordinate, + /// The witness itself. + pub witness: DurableWitness, +} + +impl LocalCausalRow for LandedWitness { fn owner(&self) -> MailboxId { - self.owner + self.witness.owner } fn cast_seq(&self) -> u64 { - self.cast_id.0 + // The DURABLE log position — monotonic across restarts — NOT `cast_id.0`. + self.coordinate.log_order() } } @@ -152,12 +217,17 @@ pub trait DurableWrite { payload: &[u8], ) -> Result; - /// Replay seam: read back every durably-landed [`DurableWitness`], in the - /// durable log's own (globally interleaved) order. Crash recovery scans this, - /// runs `temporal` layer-1 to split it per owner, and re-applies pending moves - /// in cast order ([`recover_and_apply`]). Reading the SAME co-located material - /// the receipts merely referenced — NOT a separate ledger. - async fn scan_witnesses(&self) -> Result, WriteFailed>; + /// Replay seam: read back durably-landed witnesses (each with its + /// [`DurableCoordinate`]), in ascending durable-log order, **strictly after** + /// `from` when given (so recovery reads only the tail past the last applied + /// coordinate, never the whole log). Crash recovery scans this, splits it per + /// owner (`temporal` layer-1), and re-applies pending moves in durable order + /// ([`recover_and_apply`]). Reading the SAME co-located material the receipts + /// merely referenced — NOT a separate ledger. + async fn scan_witnesses( + &self, + from: Option, + ) -> Result, WriteFailed>; } /// A **detached** cast envelope — the thinker's report, carrying its OWN payload @@ -167,13 +237,15 @@ pub trait DurableWrite { pub struct PersistCast { /// The mailbox this write is on behalf of. pub owner: MailboxId, - /// The cast this write realises — the owner-local replay order (from - /// `BatchWriter::cast`). Rides into the [`DurableWitness`] for crash replay. + /// The cast this write realises (from `BatchWriter::cast`). Rides into the + /// [`DurableWitness`] as provenance (NOT the replay-order key). pub cast_id: CastId, /// The owner-local cycle at cast time — rides into the witness (audit). pub cycle: u32, /// The lifecycle move the thought cast with this write (its `to` is applied - /// post-durability). `None` when the cast carries no lifecycle intent. + /// post-durability). `None` when the cast carries no lifecycle intent. When + /// present, `paired_move.mailbox` MUST equal `owner` (checked in + /// [`persist_cast`]). pub paired_move: Option, /// The bytes to persist — an owned/independent buffer (the concrete sink forms /// one Arrow `RecordBatch` with the witness as a second column, one generation). @@ -215,23 +287,51 @@ pub enum PersistError { /// The paired move is not a legal Rubicon edge from the owner's current phase — /// surfaced, never silently applied. Illegal(RubiconTransitionError), - /// A receipt was applied to the wrong owner — refused (a receipt only advances - /// the mailbox it was minted for; write-on-behalf never crosses owners). + /// A receipt (or the move it carries) was applied to the wrong owner — refused + /// (a receipt only advances the mailbox it was minted for; write-on-behalf + /// never crosses owners). Also raised at persist time when + /// `paired_move.mailbox != owner`, so a cross-owner move never becomes durable. OwnerMismatch { receipt_owner: MailboxId, applying_to: MailboxId, }, - /// The paired move's `from` does not match the owner's current phase — the - /// move is stale (already applied, or out of cast order). Surfaced on the - /// synchronous single-receipt path ([`apply_durable_step`]) so a mis-ordered - /// or double apply is loud; the crash-recovery path ([`recover_and_apply`]) - /// instead SKIPS a stale move (it is already reflected in the durable state). + /// The paired move's `from` does not match the owner's current phase. On the + /// synchronous single-receipt path ([`apply_durable_step`]) this is a stale / + /// out-of-order apply — surfaced, and SAFE to drop because the move is durable + /// and [`recover_and_apply`] will replay it. In recovery it means an + /// above-watermark witness does not chain — a genuine gap/corruption. StalePhase { owner_phase: KanbanColumn, move_from: KanbanColumn, }, } +impl std::fmt::Display for PersistError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Write(e) => write!(f, "{e}"), + Self::Illegal(e) => { + write!(f, "illegal Rubicon transition {:?} -> {:?}", e.from, e.to) + } + Self::OwnerMismatch { + receipt_owner, + applying_to, + } => write!( + f, + "move for mailbox {receipt_owner} applied to mailbox {applying_to}" + ), + Self::StalePhase { + owner_phase, + move_from, + } => write!( + f, + "stale move: owner at {owner_phase:?}, move.from {move_from:?}" + ), + } + } +} +impl std::error::Error for PersistError {} + /// **Phase 1 — async persistence, NO owner borrow.** Form the crash-durable /// [`DurableWitness`] and append it CO-LOCATED with the cast's payload in one /// generation; on success return a [`DurableReceipt`] (which merely references @@ -239,10 +339,22 @@ pub enum PersistError { /// receipt**. `O` (the owner) does not appear in this signature at all — it is /// never borrowed across the WAL / object-store await, so the owner stays free /// while durability runs. +/// +/// Rejects a `paired_move` minted for a different mailbox +/// ([`PersistError::OwnerMismatch`]) BEFORE the append, so a cross-owner move +/// never becomes durable (the write-on-behalf invariant, enforced at the source). pub async fn persist_cast( sink: &W, cast: PersistCast, ) -> Result { + if let Some(mv) = cast.paired_move { + if mv.mailbox != cast.owner { + return Err(PersistError::OwnerMismatch { + receipt_owner: mv.mailbox, + applying_to: cast.owner, + }); + } + } let witness = DurableWitness { owner: cast.owner, cast_id: cast.cast_id, @@ -270,6 +382,14 @@ pub async fn persist_cast( /// Returns `Ok(Some(step))` (paired move applied), `Ok(None)` (receipt carried no /// move — a durable no-step), or an error. The transition target is /// `receipt.paired_move.to` — never a generic successor. +/// +/// **Reordered / stale receipts are safe to drop.** `DurableWrite` explicitly +/// supports concurrent drains, so a later append can complete first; applying its +/// receipt while the owner is still at the earlier phase yields +/// [`PersistError::StalePhase`]. Dropping it loses NOTHING — the move is durable, +/// and [`recover_and_apply`] (or a re-drive that re-reads the durable tail) +/// replays it in durable order. The sync path is the fast happy path; the durable +/// witness is the correctness backstop. pub fn apply_durable_step( owner: &mut O, receipt: DurableReceipt, @@ -282,10 +402,17 @@ pub fn apply_durable_step( } match receipt.paired_move { Some(mv) => { + // Defense in depth: the move must also be minted for this owner (the + // envelope check above only compares receipt.owner). + if mv.mailbox != owner.mailbox_id() { + return Err(PersistError::OwnerMismatch { + receipt_owner: mv.mailbox, + applying_to: owner.mailbox_id(), + }); + } // The move's `from` must match the owner's current phase: on the - // synchronous path a mismatch is a stale / out-of-cast-order apply - // and is surfaced loudly (a double apply would otherwise silently - // re-run through `try_advance_phase`). Enforces cast order. + // synchronous path a mismatch is a stale / out-of-order apply and is + // surfaced (safe to drop — the durable witness replays it). if mv.from != owner.phase() { return Err(PersistError::StalePhase { owner_phase: owner.phase(), @@ -301,19 +428,42 @@ pub fn apply_durable_step( } } -/// **Crash recovery.** Given the durable witnesses read back via +/// The result of a [`recover_and_apply`] pass: the moves actually applied, and the +/// new durable **watermark** the caller must persist WITH the recovered SoA state +/// so the next recovery is idempotent (skips everything at or below it). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Recovered { + /// The paired moves applied this pass, in durable order. + pub applied: Vec, + /// The highest durable coordinate now accounted for (the new watermark), or + /// the input `applied_through` when nothing was applied. Persist it with the + /// owner's phase (same generation) — see the module doc § Replay order. + pub watermark: Option, +} + +/// **Crash recovery.** Given landed witnesses read back via /// [`DurableWrite::scan_witnesses`] (a globally-interleaved stream from every -/// owner) and an `owner` reconstructed at its durable phase, re-apply the owner's -/// PENDING paired moves in cast order and return them. +/// owner), an `owner` reconstructed at its durable phase, and the durable +/// `applied_through` watermark persisted alongside that phase, re-apply the +/// owner's PENDING tail in **durable-log order** and return the applied moves plus +/// the new watermark. +/// +/// - `temporal` layer-1 ([`local_trajectory_of`]) deinterlaces the global stream +/// to this owner's own chain, ordered by the durable [`DurableCoordinate`] (NOT +/// the resettable `cast_id`). +/// - Every witness at or below `applied_through` is **already reflected in the +/// recovered SoA state** and is skipped. This watermark — not phase equality — +/// is the idempotence key: the Rubicon lifecycle is cyclic, so after a completed +/// lap the owner is back at `Planning`, and a phase-only check would replay the +/// whole lap (`E-…-NOT-IN-MEMORY-ONLY-1`). +/// - Above the watermark the chain must be contiguous: a move whose `from` does +/// not match the owner's current phase is a genuine gap/corruption and is +/// surfaced ([`PersistError::StalePhase`]); a matching-`from` move that is not a +/// legal Rubicon edge is [`PersistError::Illegal`]. /// -/// The read is `temporal` layer-1: [`local_trajectory_of`] deinterlaces the -/// global stream down to this owner's own chain, in `cast_id` order — the exact -/// replay order. Each move is applied only when its `from` matches the owner's -/// current phase; a move whose `from` no longer matches is **already reflected in -/// the recovered durable SoA state** and is SKIPPED (idempotent — re-running -/// recovery after catching up applies nothing). A move that matches `from` but is -/// not a legal Rubicon edge is a genuine corruption and is surfaced -/// ([`PersistError::Illegal`]). +/// **On error the owner is left mid-chain** (every earlier move in this pass is +/// already applied). The returned `Recovered` is only produced on full success; +/// re-drive from the persisted watermark after resolving the corruption. /// /// This is why the paired move MUST be co-located in the durable generation: the /// witnesses are the only reason a KanbanStep survives a crash between the WAL @@ -321,24 +471,43 @@ pub fn apply_durable_step( /// is lost (the gap this reshape closes). pub fn recover_and_apply( owner: &mut O, - witnesses: &[DurableWitness], -) -> Result, PersistError> { - let chain = local_trajectory_of(witnesses, owner.mailbox_id()); + landed: &[LandedWitness], + applied_through: Option, +) -> Result { + let chain = local_trajectory_of(landed, owner.mailbox_id()); + let hw = applied_through.map(|c| c.log_order()); let mut applied = Vec::new(); - for w in chain { - let Some(mv) = w.paired_move else { continue }; - if mv.from != owner.phase() { - // Stale: already reflected in the recovered durable state. Skip - // (recovery is idempotent), do not surface — unlike the synchronous - // single-receipt path, a stale move here is expected, not an error. + let mut watermark = applied_through; + for lw in chain { + // Skip everything at or below the durable watermark — already reflected in + // the recovered SoA state (the cyclic-safe idempotence key). + if hw.is_some_and(|hw| lw.coordinate.log_order() <= hw) { continue; } - let step = owner - .try_advance_phase(mv.to) - .map_err(PersistError::Illegal)?; - applied.push(step); + match lw.witness.paired_move { + None => { + // A durable no-step still advances the watermark past this + // generation (it is accounted for; nothing to apply). + watermark = Some(lw.coordinate); + } + Some(mv) => { + // Above the watermark the chain MUST be contiguous: a non-matching + // `from` is a gap/corruption, not a benign already-applied move. + if mv.from != owner.phase() { + return Err(PersistError::StalePhase { + owner_phase: owner.phase(), + move_from: mv.from, + }); + } + let step = owner + .try_advance_phase(mv.to) + .map_err(PersistError::Illegal)?; + applied.push(step); + watermark = Some(lw.coordinate); + } + } } - Ok(applied) + Ok(Recovered { applied, watermark }) } #[cfg(test)] @@ -347,6 +516,8 @@ mod tests { use lance_graph_contract::collapse_gate::MailboxId as MId; use lance_graph_contract::kanban::{ExecTarget, KanbanColumn}; use lance_graph_contract::soa_view::{MailboxSoaOwner, MailboxSoaView}; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::Mutex; /// Minimal in-RAM owner (mirrors `kanban_actor::tests::TestBoard`). struct FakeOwner { @@ -399,23 +570,29 @@ mod tests { } /// A `DurableWrite` whose success/failure and call-count are observable, and - /// which RECORDS every witness it lands so [`scan_witnesses`] can replay them - /// — this is what makes the crash-recovery falsifier real: the durable - /// generation (here, `landed`) outlives the in-memory receipts. `&self` + - /// interior mutability, mirroring the real sink's shared shape. + /// which RECORDS every witness it lands (with the durable coordinate it + /// assigns) so [`scan_witnesses`] can replay them — this is what makes the + /// crash-recovery falsifier real: the durable log (`landed`) outlives the + /// in-memory receipts. `Sync` (Mutex + Atomic) so the documented concurrent + /// drain can actually be exercised. struct FakeSink { succeed: bool, - calls: std::cell::Cell, - landed: std::cell::RefCell>, + calls: AtomicU32, + /// The durable log: (coordinate, witness), assigned a monotonic WAL + /// position that does NOT reset across simulated writer lifetimes. + landed: Mutex>, } impl FakeSink { fn new(succeed: bool) -> Self { Self { succeed, - calls: std::cell::Cell::new(0), - landed: std::cell::RefCell::new(Vec::new()), + calls: AtomicU32::new(0), + landed: Mutex::new(Vec::new()), } } + fn calls(&self) -> u32 { + self.calls.load(Ordering::SeqCst) + } } impl DurableWrite for FakeSink { async fn append( @@ -423,66 +600,82 @@ mod tests { witness: &DurableWitness, _payload: &[u8], ) -> Result { - self.calls.set(self.calls.get() + 1); - if self.succeed { - // The witness lands CO-LOCATED and DURABLE — it survives even if - // every in-memory receipt is dropped (the crash the reshape guards). - self.landed.borrow_mut().push(witness.clone()); - Ok(DurableCoordinate { - shard: 0xABCD, - writer_epoch: 1, - wal_entry_position: self.landed.borrow().len() as u64, - }) - } else { + self.calls.fetch_add(1, Ordering::SeqCst); + if !self.succeed { // The negative half: a fenced WAL lands NOTHING — no witness, so // nothing to replay, so no move and no step. - Err(WriteFailed("wal fenced".into())) + return Err(WriteFailed("wal fenced".into())); } + let mut log = self.landed.lock().unwrap(); + // WAL position: monotonic over the log's whole life, 1-based. + let coordinate = DurableCoordinate { + shard: 0xABCD, + writer_epoch: 1, + wal_entry_position: log.len() as u64 + 1, + }; + log.push(LandedWitness { + coordinate, + witness: witness.clone(), + }); + Ok(coordinate) } - async fn scan_witnesses(&self) -> Result, WriteFailed> { - Ok(self.landed.borrow().clone()) + async fn scan_witnesses( + &self, + from: Option, + ) -> Result, WriteFailed> { + let lb = from.map(|c| c.log_order()); + Ok(self + .landed + .lock() + .unwrap() + .iter() + .filter(|lw| lb.is_none_or(|lb| lw.coordinate.log_order() > lb)) + .cloned() + .collect()) } } fn owner(phase: KanbanColumn) -> FakeOwner { + owner_id(42, phase) + } + fn owner_id(id: MId, phase: KanbanColumn) -> FakeOwner { FakeOwner { - id: 42, + id, phase, cycle: 5, } } + fn mv(owner: MId, from: KanbanColumn, to: KanbanColumn) -> KanbanMove { + KanbanMove { + mailbox: owner, + from, + to, + witness_chain_position: 0, + exec: ExecTarget::Elixir, + } + } fn cast(paired_to: Option) -> PersistCast { - cast_from(KanbanColumn::Planning, paired_to) + cast_of(42, 0, KanbanColumn::Planning, paired_to) } - fn cast_from(from: KanbanColumn, paired_to: Option) -> PersistCast { + fn cast_of( + owner: MId, + cast_id: u64, + from: KanbanColumn, + paired_to: Option, + ) -> PersistCast { PersistCast { - owner: 42, - cast_id: CastId(0), - cycle: 5, - paired_move: paired_to.map(|to| KanbanMove { - mailbox: 42, - from, - to, - witness_chain_position: 0, - exec: ExecTarget::Elixir, - }), - payload: vec![1, 2, 3, 4], + owner, + cast_id: CastId(cast_id), + cycle: 0, + paired_move: paired_to.map(|to| mv(owner, from, to)), + payload: vec![cast_id as u8], } } - fn receipt_for(paired_to: Option) -> DurableReceipt { - receipt_from(KanbanColumn::Planning, paired_to) - } fn receipt_from(from: KanbanColumn, paired_to: Option) -> DurableReceipt { DurableReceipt { owner: 42, cast_id: CastId(0), - paired_move: paired_to.map(|to| KanbanMove { - mailbox: 42, - from, - to, - witness_chain_position: 0, - exec: ExecTarget::Elixir, - }), + paired_move: paired_to.map(|to| mv(42, from, to)), coordinate: DurableCoordinate { shard: 0xABCD, writer_epoch: 1, @@ -490,6 +683,9 @@ mod tests { }, } } + fn receipt_for(paired_to: Option) -> DurableReceipt { + receipt_from(KanbanColumn::Planning, paired_to) + } // ── Phase 1: async persistence, NO owner borrow ──────────────────────────── @@ -509,17 +705,28 @@ mod tests { "the durable coordinate rides (first witness landed)" ); assert_eq!(r.cast_id, CastId(0), "the receipt mirrors the cast id"); - assert_eq!(sink.calls.get(), 1, "the write was attempted once"); + assert_eq!(sink.calls(), 1, "the write was attempted once"); // The witness LANDED durably (co-located) — not only in the receipt. - let scanned = sink.scan_witnesses().await.expect("scan"); + let scanned = sink.scan_witnesses(None).await.expect("scan"); assert_eq!(scanned.len(), 1, "one witness durably co-located"); assert_eq!( - scanned[0].paired_move.map(|m| m.to), + scanned[0].witness.paired_move.map(|m| m.to), Some(KanbanColumn::CognitiveWork), "the paired move is in the DURABLE witness, not only the in-memory receipt", ); } + #[tokio::test] + async fn a_failed_write_yields_no_receipt() { + let sink = FakeSink::new(false); + let r = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; + assert_eq!( + r, + Err(PersistError::Write(WriteFailed("wal fenced".into()))) + ); + assert_eq!(sink.calls(), 1, "the write was attempted"); + } + #[tokio::test] async fn a_fenced_write_lands_no_witness_so_nothing_can_be_replayed() { // Negative half, at the durable layer: a failed append records NO witness, @@ -527,23 +734,79 @@ mod tests { let sink = FakeSink::new(false); let _ = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; assert_eq!( - sink.scan_witnesses().await.expect("scan").len(), + sink.scan_witnesses(None).await.expect("scan").len(), 0, "a fenced WAL leaves no durable witness — no move, no step", ); } #[tokio::test] - async fn a_failed_write_yields_no_receipt() { - // The negative half: no durable write ⇒ NO receipt exists. There is - // therefore nothing to hand to apply_durable_step ⇒ no step can occur. - let sink = FakeSink::new(false); - let r = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; + async fn a_cross_owner_paired_move_is_rejected_before_it_becomes_durable() { + // A move minted for mailbox 99 riding in a cast on behalf of 42 must be + // refused at persist time — a cross-owner move never lands durably. + let sink = FakeSink::new(true); + let cast = PersistCast { + owner: 42, + cast_id: CastId(0), + cycle: 0, + paired_move: Some(mv(99, KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + payload: vec![], + }; assert_eq!( - r, - Err(PersistError::Write(WriteFailed("wal fenced".into()))) + persist_cast(&sink, cast).await, + Err(PersistError::OwnerMismatch { + receipt_owner: 99, + applying_to: 42 + }), + ); + assert_eq!(sink.calls(), 0, "the write was never attempted"); + assert_eq!( + sink.scan_witnesses(None).await.expect("scan").len(), + 0, + "nothing durable", + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn concurrent_drains_both_land_durably() { + // The DurableWrite doc claims &self is shared because many casts drain + // concurrently. Exercise it: two persist_cast futures on a shared sink. + let sink = std::sync::Arc::new(FakeSink::new(true)); + let s1 = sink.clone(); + let s2 = sink.clone(); + let (a, b) = tokio::join!( + async move { + persist_cast( + &*s1, + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + ) + .await + }, + async move { + persist_cast( + &*s2, + cast_of( + 43, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + ) + .await + }, + ); + assert!(a.is_ok() && b.is_ok(), "both drains landed"); + assert_eq!(sink.calls(), 2); + assert_eq!( + sink.scan_witnesses(None).await.expect("scan").len(), + 2, + "both witnesses durably recorded under concurrent drain", ); - assert_eq!(sink.calls.get(), 1, "the write was attempted"); } // ── Phase 2: synchronous owner-local completion, NO await ─────────────────── @@ -606,11 +869,7 @@ mod tests { #[test] fn a_receipt_is_refused_for_a_foreign_owner() { // A receipt minted for mailbox 42 must never advance mailbox 99. - let mut other = FakeOwner { - id: 99, - phase: KanbanColumn::Planning, - cycle: 5, - }; + let mut other = owner_id(99, KanbanColumn::Planning); let r = apply_durable_step(&mut other, receipt_for(Some(KanbanColumn::CognitiveWork))); assert!(matches!( r, @@ -626,51 +885,61 @@ mod tests { ); } + #[test] + fn a_stale_move_on_the_sync_path_is_surfaced_not_silently_reapplied() { + // The sync path surfaces a stale/out-of-order move LOUDLY (safe to drop — + // the durable witness replays it). A `Planning→…` move applied to an owner + // already at `CognitiveWork` is refused, owner untouched. + let mut o = owner(KanbanColumn::CognitiveWork); + let r = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::CognitiveWork))); + assert_eq!( + r, + Err(PersistError::StalePhase { + owner_phase: KanbanColumn::CognitiveWork, + move_from: KanbanColumn::Planning, + }), + ); + assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "owner untouched"); + } + // ── Crash recovery: the co-located witness replays after the receipt is lost ─ - /// Build a `PersistCast` for owner `owner` with an explicit cast id and move. - fn cast_of(owner: MId, cast_id: u64, from: KanbanColumn, to: KanbanColumn) -> PersistCast { - PersistCast { - owner, - cast_id: CastId(cast_id), - cycle: 0, - paired_move: Some(KanbanMove { - mailbox: owner, - from, - to, - witness_chain_position: cast_id as u32, - exec: ExecTarget::Elixir, - }), - payload: vec![cast_id as u8], + /// Persist a chain of casts through the sink (they land durably) and return the + /// scanned landed witnesses — the "durable log" a restart reads back. + async fn persist_and_scan(sink: &FakeSink, casts: Vec) -> Vec { + for c in casts { + persist_cast(sink, c).await.expect("landed"); } + sink.scan_witnesses(None).await.expect("scan") } - /// THE crash falsifier (operator): WAL append succeeds, then the process - /// dies BEFORE the sync step — every in-memory [`DurableReceipt`] is dropped. - /// On restart the owner is reconstructed at its durable phase and - /// [`recover_and_apply`] replays the PAIRED move from the co-located witness. - /// Without the reshape the move lived only in the dropped receipt and the - /// KanbanStep would be lost; here it is reconstructed and applied. + /// THE crash falsifier (operator): WAL append succeeds, then the process dies + /// BEFORE the sync step — every in-memory [`DurableReceipt`] is dropped. On + /// restart the owner is reconstructed at its durable phase (+ its durable + /// watermark) and [`recover_and_apply`] replays the PAIRED move from the + /// co-located witness. Idempotent: a second recovery from the returned + /// watermark applies nothing. #[tokio::test] async fn a_crash_after_durable_write_replays_the_move_from_the_witness() { let sink = FakeSink::new(true); - // The write lands durably (witness co-located). let receipt = persist_cast( &sink, - cast_of(42, 0, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), ) .await .expect("write landed"); - // ── CRASH ── drop the receipt without ever calling apply_durable_step. - drop(receipt); + drop(receipt); // ── CRASH ── never called apply_durable_step. - // Restart: reconstruct the owner at its DURABLE phase (Planning — the - // step never applied) and recover from what durable storage holds. - let scanned = sink.scan_witnesses().await.expect("scan"); - let mut o = owner(KanbanColumn::Planning); - let applied = recover_and_apply(&mut o, &scanned).expect("recovery legal"); + let landed = sink.scan_witnesses(None).await.expect("scan"); + let mut o = owner(KanbanColumn::Planning); // durable phase; no watermark yet + let rec = recover_and_apply(&mut o, &landed, None).expect("recovery legal"); assert_eq!( - applied.iter().map(|m| m.to).collect::>(), + rec.applied.iter().map(|m| m.to).collect::>(), vec![KanbanColumn::CognitiveWork], "the pending move was reconstructed from the durable witness and applied", ); @@ -679,41 +948,59 @@ mod tests { KanbanColumn::CognitiveWork, "the step fired on recovery" ); + assert!(rec.watermark.is_some(), "a new watermark to persist"); - // Idempotent: recovery re-run after catching up applies nothing (the move - // is now reflected in the recovered state — stale, skipped, not errored). - let again = recover_and_apply(&mut o, &scanned).expect("idempotent"); - assert!(again.is_empty(), "second recovery is a no-op"); + // Idempotent: recovery re-run FROM THE WATERMARK applies nothing. + let again = recover_and_apply(&mut o, &landed, rec.watermark).expect("idempotent"); + assert!(again.applied.is_empty(), "second recovery is a no-op"); assert_eq!(o.phase(), KanbanColumn::CognitiveWork); } - /// FALSIFIER (operator): one owner's durable batch replays its moves in - /// CAST order, and the globally-interleaved OTHER owner's witnesses are - /// deinterlaced away (temporal layer-1). The durable log interleaves owner - /// 42 and owner 99; recovery of 42 applies only 42's chain, in cast order. + /// FALSIFIER (operator): one owner's durable batch replays in DURABLE order, + /// and the globally-interleaved OTHER owner's witnesses are deinterlaced away + /// (temporal layer-1). A durable no-step interleaved into the chain is skipped + /// while the following move still applies. #[tokio::test] - async fn recovery_replays_one_owners_batch_in_cast_order_ignoring_interleaved_owners() { + async fn recovery_replays_one_owners_batch_in_order_skipping_no_steps_and_other_owners() { let sink = FakeSink::new(true); - // Global durable-log order interleaves two owners: + // Durable-log order interleaves two owners + a no-step for 42: // c0: 42 Planning→CognitiveWork // c1: 99 Planning→CognitiveWork (other owner, interleaved) - // c2: 42 CognitiveWork→Evaluation - for c in [ - cast_of(42, 0, KanbanColumn::Planning, KanbanColumn::CognitiveWork), - cast_of(99, 1, KanbanColumn::Planning, KanbanColumn::CognitiveWork), - cast_of(42, 2, KanbanColumn::CognitiveWork, KanbanColumn::Evaluation), - ] { - persist_cast(&sink, c).await.expect("landed"); - } - let scanned = sink.scan_witnesses().await.expect("scan"); - assert_eq!(scanned.len(), 3, "all three witnesses are durable"); + // c2: 42 no-step (paired_move None) + // c3: 42 CognitiveWork→Evaluation + let landed = persist_and_scan( + &sink, + vec![ + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + cast_of( + 99, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + cast_of(42, 1, KanbanColumn::Planning, None), + cast_of( + 42, + 2, + KanbanColumn::CognitiveWork, + Some(KanbanColumn::Evaluation), + ), + ], + ) + .await; + assert_eq!(landed.len(), 4, "all four witnesses are durable"); let mut o = owner(KanbanColumn::Planning); // owner 42 at its durable phase - let applied = recover_and_apply(&mut o, &scanned).expect("legal"); + let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); assert_eq!( - applied.iter().map(|m| m.to).collect::>(), + rec.applied.iter().map(|m| m.to).collect::>(), vec![KanbanColumn::CognitiveWork, KanbanColumn::Evaluation], - "owner 42's OWN chain, in cast order — 99's interleaved cast deinterlaced away", + "owner 42's OWN chain, in durable order — 99's cast + the no-step skipped", ); assert_eq!( o.phase(), @@ -722,51 +1009,190 @@ mod tests { ); } + /// THE cyclic idempotence falsifier (Codex/CodeRabbit Critical). A full lap + /// `Planning → CognitiveWork → Evaluation → Plan → Planning` leaves the owner + /// back at `Planning`. Phase equality alone would replay the whole lap; the + /// durable WATERMARK makes the second recovery a no-op. The negative control + /// (recovering WITHOUT the persisted watermark) proves the watermark is + /// load-bearing by reproducing the double-lap. + #[tokio::test] + async fn cyclic_recovery_is_idempotent_only_with_the_durable_watermark() { + // Precondition: the lap is a real cycle in the Rubicon graph. + assert_eq!(KanbanColumn::Plan.next_phases(), &[KanbanColumn::Planning]); + let sink = FakeSink::new(true); + let landed = persist_and_scan( + &sink, + vec![ + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + cast_of( + 42, + 1, + KanbanColumn::CognitiveWork, + Some(KanbanColumn::Evaluation), + ), + cast_of(42, 2, KanbanColumn::Evaluation, Some(KanbanColumn::Plan)), + cast_of(42, 3, KanbanColumn::Plan, Some(KanbanColumn::Planning)), + ], + ) + .await; + + let mut o = owner(KanbanColumn::Planning); + let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); + assert_eq!(rec.applied.len(), 4, "the whole lap applied once"); + assert_eq!( + o.phase(), + KanbanColumn::Planning, + "back at Planning after a lap" + ); + + // WITH the persisted watermark: second recovery skips the entire lap. + let good = recover_and_apply(&mut o, &landed, rec.watermark).expect("legal"); + assert!( + good.applied.is_empty(), + "watermark makes cyclic recovery idempotent", + ); + assert_eq!(o.phase(), KanbanColumn::Planning); + + // NEGATIVE CONTROL: WITHOUT the watermark (as if it were not persisted), + // phase equality replays the whole lap — the bug the watermark fixes. + let bug = recover_and_apply(&mut o, &landed, None).expect("legal"); + assert_eq!( + bug.applied.len(), + 4, + "without the durable watermark the cyclic lap is replayed — watermark is load-bearing", + ); + } + + /// FALSIFIER (Bugbot High): `CastId` resets across writer restarts, so two + /// lifetimes can share `cast_id` 0. Replay MUST order by the durable WAL + /// position (which does not reset), not `cast_id`. Two lifetimes each cast + /// `cast_id 0`, at distinct WAL positions, forming one owner's chain — recovery + /// applies them in durable order regardless of the colliding cast ids. + #[tokio::test] + async fn recovery_orders_by_durable_position_not_the_resettable_cast_id() { + let sink = FakeSink::new(true); + // Lifetime 1 (cast_id 0) then lifetime 2 (cast_id 0 again) — the counter + // reset. Distinct WAL positions (1, 2) come from the durable log. + let landed = persist_and_scan( + &sink, + vec![ + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + cast_of( + 42, + 0, + KanbanColumn::CognitiveWork, + Some(KanbanColumn::Evaluation), + ), + ], + ) + .await; + assert_eq!( + landed + .iter() + .map(|l| l.witness.cast_id.0) + .collect::>(), + vec![0, 0], + "the cast ids collide (counter reset across lifetimes)", + ); + assert_eq!( + landed + .iter() + .map(|l| l.coordinate.wal_entry_position) + .collect::>(), + vec![1, 2], + "but the durable WAL positions are distinct + monotonic", + ); + + let mut o = owner(KanbanColumn::Planning); + let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); + assert_eq!( + rec.applied.iter().map(|m| m.to).collect::>(), + vec![KanbanColumn::CognitiveWork, KanbanColumn::Evaluation], + "ordered by durable position despite the colliding cast ids", + ); + assert_eq!(o.phase(), KanbanColumn::Evaluation); + } + /// FALSIFIER 5 (operator): the durable proof is the coordinate, NOT a - /// `LanceVersion`. A witness is queryable/replayable the instant it lands — - /// before any base manifest version attaches — so recovery identifies the - /// latest LOCAL state from the co-located witness cast order, not a dataset - /// version. Here no dataset version exists at all, yet recovery is exact. + /// `LanceVersion`. A witness is replayable the instant it lands — before any + /// base manifest version attaches — so recovery identifies the latest LOCAL + /// state from the co-located witness's durable position, not a dataset version. #[tokio::test] - async fn recovery_uses_cast_order_not_a_dataset_version() { + async fn recovery_uses_durable_position_not_a_dataset_version() { let sink = FakeSink::new(true); let receipt = persist_cast( &sink, - cast_of(42, 0, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), ) .await .expect("landed"); // The durability proof carries no dataset version — only a WAL/LSM // coordinate. (The type has no `DatasetVersion` field to read.) assert!( - receipt.coordinate.wal_entry_position > 0, + receipt.coordinate.log_order() > 0, "durable via the WAL coordinate, before any manifest version", ); - let scanned = sink.scan_witnesses().await.expect("scan"); + let landed = sink.scan_witnesses(None).await.expect("scan"); let mut o = owner(KanbanColumn::Planning); - let applied = recover_and_apply(&mut o, &scanned).expect("legal"); + let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); assert_eq!( - applied.len(), + rec.applied.len(), 1, - "latest local state found from cast order alone" + "latest local state found from the durable position alone" ); } - /// The synchronous single-receipt path surfaces a stale/out-of-order move - /// LOUDLY ([`PersistError::StalePhase`]) — the counterpart to recovery's - /// silent stale-skip. A receipt whose move is `Planning→…` applied to an - /// owner already at `CognitiveWork` is refused, owner untouched. - #[test] - fn a_stale_move_on_the_sync_path_is_surfaced_not_silently_reapplied() { - let mut o = owner(KanbanColumn::CognitiveWork); - let r = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::CognitiveWork))); + /// The bounded scan seam: `scan_witnesses(from)` returns only the tail past a + /// coordinate — recovery need not read the whole log. + #[tokio::test] + async fn scan_from_a_coordinate_returns_only_the_tail() { + let sink = FakeSink::new(true); + let _ = persist_and_scan( + &sink, + vec![ + cast_of( + 42, + 0, + KanbanColumn::Planning, + Some(KanbanColumn::CognitiveWork), + ), + cast_of( + 42, + 1, + KanbanColumn::CognitiveWork, + Some(KanbanColumn::Evaluation), + ), + cast_of(42, 2, KanbanColumn::Evaluation, Some(KanbanColumn::Commit)), + ], + ) + .await; + let after_first = DurableCoordinate { + shard: 0xABCD, + writer_epoch: 1, + wal_entry_position: 1, + }; + let tail = sink.scan_witnesses(Some(after_first)).await.expect("scan"); assert_eq!( - r, - Err(PersistError::StalePhase { - owner_phase: KanbanColumn::CognitiveWork, - move_from: KanbanColumn::Planning, - }), + tail.iter() + .map(|l| l.coordinate.wal_entry_position) + .collect::>(), + vec![2, 3], + "only witnesses strictly after the given coordinate", ); - assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "owner untouched"); } } diff --git a/crates/lance-graph-planner/src/temporal.rs b/crates/lance-graph-planner/src/temporal.rs index e384d0b3..8e6ea722 100644 --- a/crates/lance-graph-planner/src/temporal.rs +++ b/crates/lance-graph-planner/src/temporal.rs @@ -397,9 +397,18 @@ where pub trait LocalCausalRow { /// The mailbox that owns this row — the deinterlace grouping key. fn owner(&self) -> MailboxId; - /// The owner-local monotonic cast sequence (e.g. `CastId.0`) — the ordering - /// key WITHIN one owner's trajectory. Cross-owner values are never compared; - /// only rows sharing an `owner()` are ordered against each other. + /// The owner-local ordering key WITHIN one owner's trajectory. Cross-owner + /// values are never compared; only rows sharing an `owner()` are ordered + /// against each other. + /// + /// PRECONDITION: **unique and monotonic per owner, ACROSS process restarts.** + /// [`local_trajectories`]'s sort is stable, so two rows of one owner sharing a + /// `cast_seq` keep their global arrival order — a tie-break that makes replay + /// order depend on scan order. A per-process counter (e.g. `BatchWriter`'s + /// `CastId`, which resets to 0 on restart) is therefore NOT a valid key: + /// different writer lifetimes collide. Use a durable-log position instead + /// (`persist_sink::LandedWitness` keys on `DurableCoordinate::log_order`, the + /// WAL entry position, which never resets). fn cast_seq(&self) -> u64; } From dcab034888de13881bc6ce53a317fecb573936cd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 00:13:54 +0000 Subject: [PATCH 6/9] fix(planner): retryable receipts + API-honest coordinate + honest test framing (PR #878 review 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChatGPT review (taken with a grain of salt — it reviewed the pre-fix c7424cd9; findings 1/2/4 were already fixed in 08fa9f3). Three further points held up: - **Reordered receipts must stay retryable, not dropped (Codex/ChatGPT).** A stale receipt dropped on the sync/happy path would only be re-applied on a crash+restart — recovery is NOT the backstop for ordinary concurrent-completion reordering. apply_durable_step now BORROWS &DurableReceipt; on StalePhase the caller keeps it and retries once the missing prefix lands. New falsifier a_reordered_receipt_is_retryable_not_lost (later receipt applied before its predecessor → neither lost, both fire in order, no crash needed). The earlier "safe to drop" doc claim was wrong and is corrected. - **API-honest durable coordinate (ChatGPT/prior note).** ShardWriter::put returns a batch position, not WalAppendResult::entry_position, so the field wal_entry_position over-promised. Renamed to an opaque backend-defined `seq` (the sink maps whichever position its API yields); no field names a WAL offset the chosen API does not return. - **Honest test/claims framing (ChatGPT; the Ladybug compile≠storage lesson).** The FakeSink tests are labelled ordering/recovery CONTRACT probes, not crash/WAL integration tests (no real MemWAL, restart, atomic RecordBatch co-location, or fencing). The fake now STORES the payload (was ignored) and a probe asserts it landed alongside the witness. Module doc + board wording downgraded from "crash-durable / high confidence" to "contract-probed; real durability unproven until the concrete sink". Taken with salt, NOT actioned as blocking: the per-cast-vs-generation persistence SEAM SHAPE (finding 5) is a real architectural fork but not a correctness bug (the operator's own bb5fcdd used per-cast) — surfaced to the operator for decision, not unilaterally reshaped. 349 planner lib tests pass; clippy + fmt clean; no Cargo.lock change. Still builds NO concrete LanceShardSink. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 4 +- .claude/board/LATEST_STATE.md | 3 +- .claude/board/STATUS_BOARD.md | 2 +- .../lance-graph-planner/src/persist_sink.rs | 226 ++++++++++++------ 4 files changed, 163 insertions(+), 72 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index c7d962ff..8bd10ffc 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,6 +1,6 @@ ## 2026-08-01 — E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1 — the persistence sink's crash gap, and temporal.rs's missing layer-1 -**Status:** FINDING (operator-ruled 2026-08-01; review-hardened — see the Correction below). **Confidence:** High — closed with falsifiable crash/recovery probes (`lance_graph_planner::persist_sink` + `temporal`); 348 planner lib tests green, clippy+fmt clean; builds NO concrete sink. +**Status:** FINDING (operator-ruled 2026-08-01; review-hardened twice — see the two Corrections below). **Confidence:** High for the **ordering/recovery CONTRACT** (349 planner lib tests, clippy+fmt clean); the crash-durability is **contract-probed over an in-process fake sink, NOT storage-proven** — real MemWAL/restart/atomic co-location durability is demonstrated only by the deferred concrete Lance sink (`compile+test green ≠ storage proven`, the Ladybug lesson). Builds NO concrete sink. **The gap the split-phase left open.** At the prior two-clock-domain shape, `persist_cast` appended only the PAYLOAD; the paired `KanbanMove` lived solely in the in-memory `DurableReceipt`. Sequence: WAL append lands (durable) → process dies before `apply_durable_step` → the receipt evaporates → the paired move is lost → the KanbanStep never fires — **even though the durable SoA state moved**. Silent: storage advanced, lifecycle did not. An in-memory receipt is not a durable record; a step that lives only in it is a step that a crash can delete. @@ -16,6 +16,8 @@ **Correction (PR #878 review, same PR — Bugbot High + Codex/CodeRabbit Critical).** The first cut ordered replay by `cast_id` and inferred "already applied" from phase equality. Both are unsound and were fixed within the PR: (a) **`CastId` resets to 0 on every writer restart** (`BatchWriter::next_id`), so cross-lifetime witnesses collide — replay now orders by the durable **`DurableCoordinate::log_order`** (WAL entry position, monotonic across restarts, never reset); `cast_id` survives only as provenance. `scan_witnesses` returns `LandedWitness{coordinate, witness}` and is the `LocalCausalRow` implementor. (b) **Phase equality is not a sound idempotence key on a CYCLIC lifecycle** (`Planning→CognitiveWork→Evaluation→Plan→Planning`): after a lap the owner is back at `Planning`, so a phase-only check replays the whole lap. `recover_and_apply` now takes a durable **watermark** (`applied_through`) — skip ≤ watermark, and above it a non-matching `from` is genuine corruption (`StalePhase`), not a benign skip; it returns the new watermark to persist WITH the SoA phase. Also: `paired_move.mailbox == owner` is validated at persist time (no cross-owner move becomes durable); `scan_witnesses(from)` is bounded; `WriteFailed`/`PersistError` impl `Display`+`Error`. New falsifiers: cyclic-idempotence with a negative control proving the watermark is load-bearing, and cast_id-collision ordering-by-durable-position. The layer-1 `cast_seq` doc now states the monotonic-across-restarts precondition. +**Correction 2 (PR #878 review — ChatGPT, taken with a grain of salt).** Three further points held up and were fixed: (a) **reordered receipts must stay retryable, NOT dropped** — a stale receipt dropped on the *happy path* would only be re-applied on a crash+restart (recovery is not the happy-path backstop for concurrent-completion reordering), so `apply_durable_step` now **borrows** `&DurableReceipt`; on `StalePhase` the caller keeps it and retries once the prefix lands (falsifier `a_reordered_receipt_is_retryable_not_lost`, no crash needed). (b) **API-honesty on the durable coordinate** — `ShardWriter::put` returns a *batch position*, not `WalAppendResult::entry_position`, so the field `wal_entry_position` was renamed to an opaque, backend-defined `seq` (the sink maps whichever its API yields); no field claims a WAL offset the chosen API does not return. (c) **honest test/claims framing** — the `FakeSink` tests are labelled **ordering/recovery CONTRACT probes**, not crash/WAL integration tests (no real WAL, restart, atomic RecordBatch co-location, or fencing); the fake now STORES the payload (was ignored) and asserts it landed alongside the witness, while the module doc states plainly that atomic co-location is unproven until the concrete sink. Points taken-with-salt and NOT actioned as blocking: the per-cast-vs-generation persistence *seam shape* (finding 5) is a real architectural fork but not a correctness bug (the operator's own `bb5fcdd` used per-cast) — surfaced to the operator for decision rather than unilaterally reshaped; a `DurableWitness::try_from_cast` constructor was deemed redundant given the persist-time guard. + ## 2026-08-01 — E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1 — the D-MBX-A6 move is pre-write intent; the KanbanStep is post-write; the persistence sink wires Lance 7's existing MemWAL and invents nothing **Status:** FINDING (operator-ruled 2026-08-01). **Confidence:** High — the pre-write half is shipped with 5 falsifiable probes (`lance_graph_planner::owner_adapter`); the persistence half's API is verified against real `lance-7.0.0` source. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index f671d09f..6ca91370 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -3,7 +3,8 @@ ### Current Contract Inventory — new/changed modules (lance-graph-planner) - `lance_graph_planner::persist_sink` — the POST-write half, two clock domains, **crash-durable**. `DurableWitness{owner, cast_id, cycle, paired_move}` is CO-LOCATED with the SoA payload in one persistence generation via `DurableWrite::append(&witness, &payload)`; the in-memory `DurableReceipt` merely REFERENCES it (via `DurableCoordinate`), never the only copy of the move. **Replay order = the durable `DurableCoordinate::log_order` (WAL position, monotonic across restarts), NOT `cast_id`** (`BatchWriter`'s counter resets on restart → cross-lifetime collisions; `cast_id` is provenance only). `DurableWrite::scan_witnesses(from)` returns `LandedWitness{coordinate, witness}` (bounded tail read; `LandedWitness` is the `LocalCausalRow` implementor). `recover_and_apply(owner, landed, applied_through)` = crash recovery: temporal layer-1 → per-owner tail in durable order; **idempotence is a durable WATERMARK** (`applied_through`), NOT phase equality (unsound on the cyclic lifecycle — a completed lap returns to `Planning`); above the watermark a non-matching `from` is corruption (`StalePhase`); returns `Recovered{applied, watermark}` to persist with the SoA phase. `persist_cast` validates `paired_move.mailbox == owner` (no cross-owner move becomes durable); `apply_durable_step` keeps the sync `from==phase` guard (a stale drop is safe — the durable witness replays). `WriteFailed`/`PersistError` impl `Display`+`Error`. Async `persist_cast` (no owner borrow) / sync `apply_durable_step` (no await) split preserved. Durability proof = `DurableCoordinate`, NEVER `LanceVersion`. **No concrete `LanceShardSink` built** (deferred per operator, gated on the crash falsifiers). - `lance_graph_planner::temporal` — **layer-1 CAUSAL deinterlacing added** (the missing half): `LocalCausalRow{owner, cast_seq}` + `local_trajectories`/`local_trajectory_of` split a globally-interleaved durable log into per-owner LOCAL chains ordered by `cast_seq` (`A@s0,C@s0,B@s0,A@s1` → A's `[A@s0,A@s1]`; interleaved owners removed). `cast_seq` doc states the monotonic-across-restarts precondition (a resettable counter is NOT valid; use a durable-log position). Composes with the pre-existing layer-2 epistemic projection (`classify`/`deinterlace`). `LandedWitness` implements `LocalCausalRow`. -- Gates: operator crash/recovery falsifiers green + layer-1 falsifiers + review-driven additions (cyclic-idempotence with negative control; cast_id-collision ordered by durable position; cross-owner reject; concurrent drains; bounded scan); 348 planner lib tests, `cargo clippy -p lance-graph-planner` + `cargo fmt -p` clean. `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1`. +- Round-2 review (ChatGPT, grain of salt): `apply_durable_step` now **borrows** `&DurableReceipt` so a reordered/stale receipt stays RETRYABLE on the happy path (not dropped-until-crash); `DurableCoordinate.wal_entry_position` renamed to opaque `seq` (API-honest — `ShardWriter::put` returns a batch position, not a WAL offset); the `FakeSink` tests are labelled **CONTRACT probes** (the fake now stores + asserts the payload) — real MemWAL/restart/atomic-co-location durability is UNPROVEN until the concrete sink (`compile+test green ≠ storage proven`). +- **Honest status:** the ordering/recovery CONTRACT is green (349 planner lib tests, clippy + fmt clean); crash-durability is contract-probed over an in-process fake, NOT storage-proven. Deferred per operator: the concrete `LanceShardSink`; and the per-cast-vs-**generation/batch** persistence seam shape (finding 5 — surfaced for operator decision, not a correctness bug). `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1`. ## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3c owner-consume adapter (rebased onto main dcd9cc9) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index a7ff8678..0270d578 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -963,7 +963,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3a | StyleStrategy: thinking-style -> cluster -> mechanism -> recipe_kernels Tactic selection (planning substrate; carries tau JIT addr) | lance-graph-planner | 130 | LOW | **In PR** | #439; first cut of A6-P3 consumer wiring; planner now consumes contract recipes/styles; deferred: i4-32D decode, Outcome->Candidate, tau->JIT, membrane commit | | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | -| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate`, never `LanceVersion`. | lance-graph-planner | 160 | LOW | **In PR** | #878; operator falsifiers green + review-hardened (Bugbot High: cast_id-resets→order by durable position; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark w/ negative control; cross-owner reject; concurrent drains; bounded scan); 348 planner lib tests, clippy+fmt clean; builds NO concrete `LanceShardSink` (deferred per operator — gated on these falsifiers); `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | +| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **In PR** | #878; ordering/recovery CONTRACT probed (349 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs index d535d108..b48b2560 100644 --- a/crates/lance-graph-planner/src/persist_sink.rs +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -54,7 +54,7 @@ //! Two distinct keys, both durable, neither the resettable `CastId` counter: //! //! - **Replay ORDER = the durable-log position** ([`DurableCoordinate::log_order`], -//! i.e. `wal_entry_position`). `BatchWriter`'s `CastId` counter **resets to 0 on +//! i.e. `seq`). `BatchWriter`'s `CastId` counter **resets to 0 on //! every restart**, so two witnesses from different writer lifetimes can share a //! `cast_id` — ordering by it is unstable across crashes. The WAL position is //! monotonic across the shard's whole life and never resets; `writer_epoch` @@ -76,30 +76,41 @@ //! //! ## The `DurableWrite` seam + the durability type //! -//! A WAL append produces a durable **coordinate** (shard + writer epoch + WAL -//! entry position), NOT a base `DatasetVersion` — the dataset version arrives +//! A durable append produces a **coordinate** (shard + writer epoch + opaque +//! monotonic `seq`), NOT a base `DatasetVersion` — the dataset version arrives //! later via MemTable flush + manifest commit. So [`DurableWrite::append`] returns //! [`DurableCoordinate`], never a version. This is exactly why the coordinate, -//! not `LanceVersion`, is the durability proof: the write is queryable and -//! durable *before* any base manifest version attaches (falsifier 5 — -//! WAL-visible-before-manifest — proves the latest local state is identified from -//! the co-located witness's durable position, not from a dataset version that -//! does not exist yet). +//! not `LanceVersion`, is the durability proof: the write is durable *before* any +//! base manifest version attaches, so the latest local state is identified from +//! the co-located witness's durable position, not from a dataset version that does +//! not exist yet. //! //! The concrete impl (a lance-having crate) wires lance 7.0.0's OFFICIAL MemWAL — //! preferring the high-level `ShardWriter::put` (`enable_memtable + durable_write`: //! insert into the queryable MemTable, release the lock, then await WAL durability -//! off-lock) over the raw `WalAppender::append` primitive. The witness and the -//! payload are two columns of ONE `RecordBatch` (one generation, atomic), whose -//! buffers are independently owned, so "no owner borrow across I/O" holds without -//! any claim of magic zero-copy. [`DurableWrite::scan_witnesses`] takes a `from` -//! lower bound so recovery reads only the tail after the last applied coordinate, -//! never the whole log. +//! off-lock) over the raw `WalAppender::append` primitive. It will co-locate the +//! witness and the payload in ONE durable generation. [`DurableWrite::scan_witnesses`] +//! takes a `from` lower bound so recovery reads only the tail after the last +//! applied coordinate, never the whole log. //! //! Keeping the seam a trait leaves the ordering core lance-free and `protoc`-free. //! **This module builds NO concrete `LanceShardSink`** — per operator ruling the //! durable-witness reshape + `temporal` layer-1 land first; the production sink -//! comes after, gated on the crash-recovery falsifiers here going green. +//! comes after. +//! +//! ## What the tests prove — CONTRACT probes, NOT crash/WAL integration (be honest) +//! +//! The `#[cfg(test)]` `FakeSink` is an in-process vector, not a WAL. So the tests +//! here are **ordering/recovery CONTRACT probes** — they prove the Rust seam +//! *reconstructs and re-applies a move from a witness once it is durable*, that +//! replay orders by the durable coordinate (not the resettable `CastId`), that +//! recovery is idempotent under a watermark across a cyclic lap, and that a failed +//! append leaves no witness. They do **NOT** prove real durability: there is no +//! real MemWAL, no process restart, no `RecordBatch` atomic co-location of witness +//! + payload, no epoch fencing, no manifest-independent read. **`compile+test green +//! ≠ storage behaviour proven`** (the Ladybug lesson, `CLAUDE.md`). Real +//! crash-durability is demonstrated only by the deferred concrete Lance sink's own +//! integration test; until then this is a *probed contract*, not a durable system. use lance_graph_contract::collapse_gate::MailboxId; use lance_graph_contract::kanban::{KanbanColumn, KanbanMove, RubiconTransitionError}; @@ -121,31 +132,42 @@ impl std::error::Error for WriteFailed {} /// A backend-neutral **durable coordinate** — where the write landed in the /// durable log. This is NOT a base Lance `DatasetVersion` (that arrives later via -/// MemTable flush + manifest commit); it is the WAL/LSM coordinate that proves -/// durability now. For lance: `shard` = shard `Uuid` (as `u128`), -/// `writer_epoch` + `wal_entry_position` from the `ShardWriter`/`WalAppendResult`. +/// MemTable flush + manifest commit); it is the durable-log coordinate that proves +/// durability now. +/// +/// **API-honest by design.** `seq` is an *opaque, backend-defined* monotonic +/// position — NOT specifically a raw WAL entry offset. Lance 7's chosen high-level +/// path (`ShardWriter::put` with `enable_memtable`) returns a **batch-position** +/// result, NOT `WalAppendResult::entry_position`; the raw `WalAppender::append` +/// returns the entry position. Both are monotonic per shard, so the concrete sink +/// maps whichever its API yields into `seq` — this type never claims a WAL offset +/// the selected API does not return. For lance: `shard` = shard `Uuid` (as `u128`), +/// `writer_epoch` = the writer session that fenced the append. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DurableCoordinate { /// Opaque shard identity (lance: the shard `Uuid` as `u128`). pub shard: u128, - /// The writer epoch that fenced this append. A new writer lifetime takes a - /// higher epoch; fencing guarantees two epochs never share a WAL position. + /// The writer session/epoch that fenced this append. A new writer lifetime + /// takes a higher epoch; fencing guarantees two epochs never share a `seq`. + /// This is the restart discriminator (the ephemeral `CastId` counter is not). pub writer_epoch: u64, - /// The monotonic WAL entry position of this durable append. Monotonic across - /// the shard's whole life — it does **not** reset on writer restart. - pub wal_entry_position: u64, + /// The **opaque backend-defined monotonic durable position** of this append + /// (raw WAL entry position, or `ShardWriter::put`'s batch position — the sink + /// picks). Monotonic across the shard's whole life; it does **not** reset on + /// writer restart. + pub seq: u64, } impl DurableCoordinate { /// The durable-log total-order key for replay + the watermark comparison. /// - /// `wal_entry_position` is monotonic across the shard's whole life and never - /// resets (unlike `BatchWriter`'s `CastId` counter), and `writer_epoch` fences - /// overlapping writers so positions never collide. One owner writes one shard, - /// so this is a total order over that owner's witnesses, valid across crashes. + /// `seq` is monotonic across the shard's whole life and never resets (unlike + /// `BatchWriter`'s `CastId` counter), and `writer_epoch` fences overlapping + /// writers so positions never collide. One owner writes one shard, so this is a + /// total order over that owner's witnesses, valid across crashes. #[must_use] pub fn log_order(&self) -> u64 { - self.wal_entry_position + self.seq } } @@ -297,8 +319,11 @@ pub enum PersistError { }, /// The paired move's `from` does not match the owner's current phase. On the /// synchronous single-receipt path ([`apply_durable_step`]) this is a stale / - /// out-of-order apply — surfaced, and SAFE to drop because the move is durable - /// and [`recover_and_apply`] will replay it. In recovery it means an + /// out-of-order apply (concurrent drains can complete a later receipt first) — + /// surfaced so the caller can **RETRY** it once the missing prefix lands. + /// [`apply_durable_step`] borrows the receipt, so it is NOT consumed on this + /// error and stays retryable (recovery only runs on restart — it is not the + /// happy-path backstop for reordered completions). In recovery it means an /// above-watermark witness does not chain — a genuine gap/corruption. StalePhase { owner_phase: KanbanColumn, @@ -383,16 +408,18 @@ pub async fn persist_cast( /// move — a durable no-step), or an error. The transition target is /// `receipt.paired_move.to` — never a generic successor. /// -/// **Reordered / stale receipts are safe to drop.** `DurableWrite` explicitly -/// supports concurrent drains, so a later append can complete first; applying its -/// receipt while the owner is still at the earlier phase yields -/// [`PersistError::StalePhase`]. Dropping it loses NOTHING — the move is durable, -/// and [`recover_and_apply`] (or a re-drive that re-reads the durable tail) -/// replays it in durable order. The sync path is the fast happy path; the durable -/// witness is the correctness backstop. +/// **Reordered receipts stay RETRYABLE — the receipt is BORROWED, not consumed.** +/// `DurableWrite` explicitly supports concurrent drains, so a later append can +/// complete first; applying its receipt while the owner is still at the earlier +/// phase yields [`PersistError::StalePhase`]. Because this takes `&receipt`, the +/// caller still owns it and re-applies it once the missing prefix lands — the +/// KanbanStep is NOT lost on the happy path (recovery only runs on restart, so it +/// is not the backstop for ordinary completion reordering). Per-owner completion +/// order is the caller's to sequence (buffer the non-contiguous tail); cross-owner +/// drains stay fully concurrent. pub fn apply_durable_step( owner: &mut O, - receipt: DurableReceipt, + receipt: &DurableReceipt, ) -> Result, PersistError> { if receipt.owner != owner.mailbox_id() { return Err(PersistError::OwnerMismatch { @@ -578,9 +605,12 @@ mod tests { struct FakeSink { succeed: bool, calls: AtomicU32, - /// The durable log: (coordinate, witness), assigned a monotonic WAL - /// position that does NOT reset across simulated writer lifetimes. - landed: Mutex>, + /// The durable log: (landed witness, payload) pairs, assigned a monotonic + /// `seq` that does NOT reset across simulated writer lifetimes. The payload + /// is STORED (not ignored) so a probe can show witness+payload land + /// together — though an in-process `Vec` cannot prove *atomic* co-location + /// (that is the concrete Lance sink's `RecordBatch`, deferred). + landed: Mutex)>>, } impl FakeSink { fn new(succeed: bool) -> Self { @@ -593,30 +623,43 @@ mod tests { fn calls(&self) -> u32 { self.calls.load(Ordering::SeqCst) } + /// Test-only: the (coordinate.seq, payload) pairs the sink holds — used to + /// show the payload landed alongside its witness, not dropped. + fn landed_payloads(&self) -> Vec<(u64, Vec)> { + self.landed + .lock() + .unwrap() + .iter() + .map(|(lw, p)| (lw.coordinate.seq, p.clone())) + .collect() + } } impl DurableWrite for FakeSink { async fn append( &self, witness: &DurableWitness, - _payload: &[u8], + payload: &[u8], ) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); if !self.succeed { - // The negative half: a fenced WAL lands NOTHING — no witness, so - // nothing to replay, so no move and no step. + // The negative half: a fenced WAL lands NOTHING — no witness, no + // payload — so nothing to replay, so no move and no step. return Err(WriteFailed("wal fenced".into())); } let mut log = self.landed.lock().unwrap(); - // WAL position: monotonic over the log's whole life, 1-based. + // Durable position: monotonic over the log's whole life, 1-based. let coordinate = DurableCoordinate { shard: 0xABCD, writer_epoch: 1, - wal_entry_position: log.len() as u64 + 1, + seq: log.len() as u64 + 1, }; - log.push(LandedWitness { - coordinate, - witness: witness.clone(), - }); + log.push(( + LandedWitness { + coordinate, + witness: witness.clone(), + }, + payload.to_vec(), + )); Ok(coordinate) } async fn scan_witnesses( @@ -629,6 +672,7 @@ mod tests { .lock() .unwrap() .iter() + .map(|(lw, _payload)| lw) .filter(|lw| lb.is_none_or(|lb| lw.coordinate.log_order() > lb)) .cloned() .collect()) @@ -679,7 +723,7 @@ mod tests { coordinate: DurableCoordinate { shard: 0xABCD, writer_epoch: 1, - wal_entry_position: 7, + seq: 7, }, } } @@ -701,7 +745,7 @@ mod tests { Some(KanbanColumn::CognitiveWork) ); assert_eq!( - r.coordinate.wal_entry_position, 1, + r.coordinate.seq, 1, "the durable coordinate rides (first witness landed)" ); assert_eq!(r.cast_id, CastId(0), "the receipt mirrors the cast id"); @@ -714,6 +758,14 @@ mod tests { Some(KanbanColumn::CognitiveWork), "the paired move is in the DURABLE witness, not only the in-memory receipt", ); + // …and the PAYLOAD landed with it at the same seq (not ignored). (An + // in-process Vec cannot prove *atomic* co-location — that is the concrete + // Lance sink's RecordBatch; the module doc states this honestly.) + assert_eq!( + sink.landed_payloads(), + vec![(1u64, vec![0u8])], + "the payload landed alongside the witness at seq 1", + ); } #[tokio::test] @@ -814,7 +866,7 @@ mod tests { #[test] fn applying_a_receipt_advances_the_owner_by_the_paired_move() { let mut o = owner(KanbanColumn::Planning); - let step = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::CognitiveWork))) + let step = apply_durable_step(&mut o, &receipt_for(Some(KanbanColumn::CognitiveWork))) .expect("legal") .expect("a paired move"); assert_eq!(step.to, KanbanColumn::CognitiveWork); @@ -835,7 +887,7 @@ mod tests { let mut o = owner(KanbanColumn::Evaluation); let step = apply_durable_step( &mut o, - receipt_from(KanbanColumn::Evaluation, Some(KanbanColumn::Prune)), + &receipt_from(KanbanColumn::Evaluation, Some(KanbanColumn::Prune)), ) .expect("legal") .expect("paired veto"); @@ -851,7 +903,7 @@ mod tests { #[test] fn a_receipt_with_no_paired_move_is_a_durable_no_step() { let mut o = owner(KanbanColumn::CognitiveWork); - let r = apply_durable_step(&mut o, receipt_for(None)); + let r = apply_durable_step(&mut o, &receipt_for(None)); assert_eq!(r, Ok(None), "durable, but no step"); assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "phase unchanged"); } @@ -861,7 +913,7 @@ mod tests { // Planning → Evaluation skips CognitiveWork — illegal. Surfaced; owner // untouched (the checked airgap holds in the owner-local phase). let mut o = owner(KanbanColumn::Planning); - let r = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::Evaluation))); + let r = apply_durable_step(&mut o, &receipt_for(Some(KanbanColumn::Evaluation))); assert!(matches!(r, Err(PersistError::Illegal(_)))); assert_eq!(o.phase(), KanbanColumn::Planning, "owner untouched"); } @@ -870,7 +922,7 @@ mod tests { fn a_receipt_is_refused_for_a_foreign_owner() { // A receipt minted for mailbox 42 must never advance mailbox 99. let mut other = owner_id(99, KanbanColumn::Planning); - let r = apply_durable_step(&mut other, receipt_for(Some(KanbanColumn::CognitiveWork))); + let r = apply_durable_step(&mut other, &receipt_for(Some(KanbanColumn::CognitiveWork))); assert!(matches!( r, Err(PersistError::OwnerMismatch { @@ -887,11 +939,11 @@ mod tests { #[test] fn a_stale_move_on_the_sync_path_is_surfaced_not_silently_reapplied() { - // The sync path surfaces a stale/out-of-order move LOUDLY (safe to drop — - // the durable witness replays it). A `Planning→…` move applied to an owner - // already at `CognitiveWork` is refused, owner untouched. + // The sync path surfaces a stale/out-of-order move LOUDLY. A `Planning→…` + // move applied to an owner already at `CognitiveWork` is refused, owner + // untouched — and the borrowed receipt stays usable (see the retry test). let mut o = owner(KanbanColumn::CognitiveWork); - let r = apply_durable_step(&mut o, receipt_for(Some(KanbanColumn::CognitiveWork))); + let r = apply_durable_step(&mut o, &receipt_for(Some(KanbanColumn::CognitiveWork))); assert_eq!( r, Err(PersistError::StalePhase { @@ -902,6 +954,47 @@ mod tests { assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "owner untouched"); } + /// FALSIFIER (Codex/ChatGPT): concurrent drains can complete a LATER receipt + /// before its predecessor. The later receipt must NOT be lost — `apply_durable_step` + /// borrows it, so on `StalePhase` the caller still owns it and retries once the + /// missing prefix lands. Both steps fire, in order, on the HAPPY path (no crash). + #[test] + fn a_reordered_receipt_is_retryable_not_lost() { + // Two casts for one owner: r8 (Planning→CognitiveWork) then + // r9 (CognitiveWork→Evaluation). r9's append completes FIRST. + let r8 = receipt_from(KanbanColumn::Planning, Some(KanbanColumn::CognitiveWork)); + let r9 = receipt_from(KanbanColumn::CognitiveWork, Some(KanbanColumn::Evaluation)); + let mut o = owner(KanbanColumn::Planning); + + // r9 arrives first — stale (owner still at Planning). Surfaced, NOT consumed. + assert!(matches!( + apply_durable_step(&mut o, &r9), + Err(PersistError::StalePhase { .. }), + )); + assert_eq!( + o.phase(), + KanbanColumn::Planning, + "r9 did not apply out of order" + ); + + // r8's append completes; it applies. + apply_durable_step(&mut o, &r8) + .expect("r8 legal") + .expect("moved"); + assert_eq!(o.phase(), KanbanColumn::CognitiveWork); + + // The caller RETRIES the still-owned r9 — it now applies. Nothing was lost, + // and no crash/recovery was needed (the happy path stays correct). + apply_durable_step(&mut o, &r9) + .expect("r9 legal now") + .expect("moved"); + assert_eq!( + o.phase(), + KanbanColumn::Evaluation, + "both steps fired, in order" + ); + } + // ── Crash recovery: the co-located witness replays after the receipt is lost ─ /// Persist a chain of casts through the sink (they land durably) and return the @@ -1105,10 +1198,7 @@ mod tests { "the cast ids collide (counter reset across lifetimes)", ); assert_eq!( - landed - .iter() - .map(|l| l.coordinate.wal_entry_position) - .collect::>(), + landed.iter().map(|l| l.coordinate.seq).collect::>(), vec![1, 2], "but the durable WAL positions are distinct + monotonic", ); @@ -1184,13 +1274,11 @@ mod tests { let after_first = DurableCoordinate { shard: 0xABCD, writer_epoch: 1, - wal_entry_position: 1, + seq: 1, }; let tail = sink.scan_witnesses(Some(after_first)).await.expect("scan"); assert_eq!( - tail.iter() - .map(|l| l.coordinate.wal_entry_position) - .collect::>(), + tail.iter().map(|l| l.coordinate.seq).collect::>(), vec![2, 3], "only witnesses strictly after the given coordinate", ); From b6f904c82d6c69662e5be8c559082c23b53b9d99 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 01:34:25 +0000 Subject: [PATCH 7/9] refactor(planner): persistence sink is WAL-amortized per cycle, not per cast (PR #878) The operator ruled the seam-shape fork surfaced in Correction 2(c) of E-THE-PAIRED-MOVE-...-1: the durable unit is the cycle/sweep, not the cast. 64k concurrent thoughts stage into an owned Vec; persist_cycle folds+freezes them; WalSink::commit_cycle does exactly ONE atomic append -> one DatasetVersion. Modelling durability per-cast paid WAL header + durability-wait + version-metadata 64k times; this amortizes it. The layer's ONLY concerns are storage + no physical/epistemic race: - Physical race: order_cycle_stably (generic over the caller's key) stable- orders casts by their EXISTING stream_position in DetachedCycleBatch::freeze BEFORE the single append. scan_sealed returns stored order and NEVER sorts. Write-side ordering lives here, NOT in temporal.rs (which owns query-time reading); the deinterlace_write additions made there this session are reverted (temporal.rs untouched). - Epistemic race: sealed read horizon (read Vn / write Vn+1); an uncommitted cycle is invisible to scan_sealed. - Coalescing: real per-row fold (row -> last payload in stream order), not last-chunk-wins. No semantic/witness/ancestry/awareness/branch/rung/temporal-policy type is minted in this layer. CycleFrame is storage-only {cycle, base_version}; SweepSlot is a boring landing (no basis). Vocabulary reused, not re-minted: DatasetVersion, KanbanMove/KanbanColumn, MailboxId/MailboxSoaOwner, stream_position. recover_and_apply + the durable watermark idempotence + StalePhase/OwnerMismatch guards survive unchanged. Retired: DurableWitness, DurableReceipt, DurableCoordinate, DurableWrite, persist_cast, apply_durable_step, scan_witnesses, LandedWitness. Tests: 13 storage/race CONTRACT probes over an in-process FakeWalSink, honestly labelled (compile+test green != storage proven, the Ladybug lesson). No concrete Lance sink built (deferred, gated on crash falsifiers). lance-graph-planner: 344 lib tests green, clippy+fmt clean. Board: EPIPHANIES E-THE-DURABLE-UNIT-IS-THE-CYCLE-...-1 + LATEST_STATE D-MBX-A6-P3e (prepended, same commit). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 18 + .claude/board/LATEST_STATE.md | 6 + .../lance-graph-planner/src/persist_sink.rs | 1622 +++++++---------- 3 files changed, 668 insertions(+), 978 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 8bd10ffc..000bc37e 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,21 @@ +## 2026-08-02 — E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1 — the persistence seam is reshaped to the WAL-amortized generation; per-cast durability retired + +**Status:** FINDING (operator-ruled 2026-08-02 — the decision on the fork Correction 2(c) of `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` surfaced). **Confidence:** High for the **storage/race CONTRACT** (`lance-graph-planner` 344 lib tests, 13 in `persist_sink`, clippy+fmt clean); crash-durability remains contract-probed over an in-process fake, NOT storage-proven (`compile+test green ≠ storage proven`, the Ladybug lesson). Builds NO concrete sink. + +**The ruling.** The durable unit is the **cycle/sweep** — one globally-aligned 64k-row frame — NOT the per-cast `DurableWitness`. `persist_cast`-per-thought would pay WAL header + durability-wait + version-metadata 64k times; the reshape **amortizes the WAL**: 64k concurrent thoughts stage into an owned cast vector, one `persist_cycle` folds+freezes them, and `WalSink::commit_cycle` does exactly **one atomic append → one `DatasetVersion`**. cast ⊂ chunk ⊂ cycle. + +**The layer's ONLY concerns are storage + that 64k concurrent thoughts hit no physical or epistemic race.** It models no semantics: it mints NO witness / ancestry / awareness / branch / rung / temporal-policy type. `CycleFrame` is storage-only `{cycle, base_version}`; `SweepSlot` is a boring landing (`where`, never `why` — no `basis`). + +**Physical race → WRITE-SIDE order, never read-time repair.** Thoughts finish in arbitrary CPU order; `DetachedCycleBatch::freeze` stable-orders casts by their EXISTING `stream_position` (`order_cycle_stably`, generic over the key) BEFORE the single append, so the durable image is already canonical. `scan_sealed` returns stored order and **never sorts** (a read-time sort would lose the whole amortization). This write-side ordering lives in `persist_sink.rs`, **NOT `temporal.rs`** — that module owns query-time reader-horizon/version reading; the write-side deinterlace additions made there this session were reverted. + +**Epistemic race → the sealed read horizon.** Every thought reads only the sealed predecessor `Vn` (`frame.base_version`); the commit publishes `Vn+1` once, all-or-nothing. `scan_sealed` excludes any uncommitted cycle, so no thought reads a concurrent sibling's in-flight output as though it were sealed history (`read Vn / write Vn+1`). + +**Coalescing is a real per-row fold, not last-chunk-wins.** The final image is `row -> last payload in stream order`; chunks are disjoint/mergeable slices of that image. + +**Vocabulary reused, not re-minted.** `DatasetVersion` (scheduler), `KanbanMove`/`KanbanColumn`/`RubiconTransitionError` (kanban), `MailboxId`/`MailboxSoaOwner`, `stream_position` (the existing canonical key). `recover_and_apply` + the durable **watermark** idempotence (not phase equality — the lifecycle is cyclic) + the `StalePhase`/`OwnerMismatch` corruption guards survive from the prior model. + +**Retired (this reshape).** `DurableWitness`, `DurableReceipt`, `DurableCoordinate`, `DurableWrite`, `persist_cast`, `apply_durable_step`, `scan_witnesses`, `LandedWitness` — the per-cast durability surface. Their intent is preserved in the cycle machinery (`CycleFrame`, `SweepSlot`, `LandedSlot`, `DetachedCycleBatch`, `WalSink::commit_cycle`, `persist_cycle`, `order_cycle_stably`). Supersedes the seam-shape half of `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` (its ordering/recovery/watermark CONTRACT stands). Still deferred: the concrete Lance sink, gated on crash falsifiers. PR #878 (reshaped in place). + ## 2026-08-01 — E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1 — the persistence sink's crash gap, and temporal.rs's missing layer-1 **Status:** FINDING (operator-ruled 2026-08-01; review-hardened twice — see the two Corrections below). **Confidence:** High for the **ordering/recovery CONTRACT** (349 planner lib tests, clippy+fmt clean); the crash-durability is **contract-probed over an in-process fake sink, NOT storage-proven** — real MemWAL/restart/atomic co-location durability is demonstrated only by the deferred concrete Lance sink (`compile+test green ≠ storage proven`, the Ladybug lesson). Builds NO concrete sink. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 6ca91370..96f4a179 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,9 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3e persistence-sink reshaped to WAL-amortized cycle (one write per sweep) + +- `lance_graph_planner::persist_sink` — **reshaped in place** (PR #878) from the per-cast durable-witness model to the **cycle/WAL** model (operator ruling on the seam-shape fork surfaced 2026-08-01). The durable unit is the **cycle/sweep**, NOT the cast: 64k concurrent thoughts stage into an owned `Vec`, `persist_cycle(sink, frame, casts)` folds+freezes them, and `WalSink::commit_cycle(base, DetachedCycleBatch)` does **exactly one atomic append → one `DatasetVersion`** (WAL amortization — 64k thoughts, one write). Types: `CycleId`, `CycleFrame{cycle, base_version}` (storage-only — NO rung/branch/semantic tags), `SweepSlot{cycle, stream_position, owner, row, paired_move, payload}` (boring landing, no `basis`), `LandedSlot{version, slot}`, `DetachedCycleBatch{frame, landings, image}` (frozen, deinterlaced, coalesced). **Write-side ordering** = `order_cycle_stably(rows, key)` (generic over the caller's canonical key; stable-orders casts by the EXISTING `stream_position` in `freeze` BEFORE the append) — completion order never becomes storage order (physical race); `scan_sealed` returns stored order and NEVER sorts. **Epistemic horizon** = sealed read (`read Vn / write Vn+1`); an uncommitted cycle is invisible to `scan_sealed`. **Coalescing** = real per-row fold (`row -> last payload in stream order`), not last-chunk-wins. `recover_and_apply(owner, sealed, applied_through)` + the durable **watermark** idempotence + `StalePhase`/`OwnerMismatch` guards survive unchanged. `versions()` = the cheap coarse timeline a downstream time-series consumer (stockfish-rs, another session) looks up, no landing replay. +- **Write-side ordering lives HERE, not in `temporal.rs`** — that module owns query-time reader-horizon/version reading; the write-side `deinterlace_write` additions made there earlier this session were **reverted** (temporal.rs is untouched — `git diff` clean). The rung a cycle aligns at is decided by whoever schedules it, never modelled in `CycleFrame`. +- **Retired:** `DurableWitness`, `DurableReceipt`, `DurableCoordinate`, `DurableWrite`, `persist_cast`, `apply_durable_step`, `scan_witnesses`, `LandedWitness` (the per-cast surface). Vocabulary reused, not re-minted: `DatasetVersion`, `KanbanMove`/`KanbanColumn`, `MailboxId`/`MailboxSoaOwner`, `stream_position`. **No concrete Lance sink built** (deferred per operator, gated on crash falsifiers). Tests are storage/race CONTRACT probes over an in-process `FakeWalSink`, honestly labelled — real MemWAL/restart/atomic-append durability UNPROVEN. See `EPIPHANIES.md` E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1. + ## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3d persistence-sink durable-witness reshape + temporal layer-1 ### Current Contract Inventory — new/changed modules (lance-graph-planner) diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs index b48b2560..46976385 100644 --- a/crates/lance-graph-planner/src/persist_sink.rs +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -1,330 +1,221 @@ -//! The D-MBX-A6 persistence sink — **two cleanly separated clock domains**. +//! The D-MBX-A6 persistence sink — **one WAL write per cycle** (the amortization). //! //! This is the POST-write half of the fire-and-forget flow (pre-write half = -//! [`crate::owner_adapter`]). The thinker already reported and moved on: +//! [`crate::owner_adapter`]). Its ONLY concerns are **storage** and that 64k +//! concurrent thoughts hit **no physical or epistemic race condition** (operator +//! ruling). It does not model semantics: semantic causality already lives in the +//! witness substrate (`causal_witness`, AriGraph SPO, NARS inference direction) — +//! this layer never invents a second causality graph. //! -//! ```text -//! thinking path: SoA → BatchWriter.cast(on_behalf, paired_move, descriptor) → keep thinking -//! persistence path: PersistCast → async durable append → DurableReceipt (NO owner borrow) -//! owner-local step: DurableReceipt + paired move → try_advance_phase (NO await) -//! ``` +//! > The sweep globally aligns **durability**; `temporal.rs` locally aligns +//! > **order** before durability. One semantic generation = one globally-aligned +//! > 64k-row **cycle** that seals into exactly one [`DatasetVersion`]. //! -//! ## Why the two phases must NOT be one async function (operator-ruled) +//! ## Three levels — cast ⊂ chunk ⊂ cycle //! -//! A monolithic `persist_then_step(&mut owner, …, bytes).await` would hold BOTH -//! `&mut owner` and `&[u8]` borrowed from live owner state **across the WAL I/O -//! await** — immobilizing the owner while the object store completes, which -//! defeats the latency masking the architecture exists to obtain. So the durable -//! write ([`persist_cast`], async, **no `O` in its signature**) and the lifecycle -//! step ([`apply_durable_step`], sync, **no await**) are split. The exclusive -//! `&mut owner` is held only in the sync phase, outside the storage-latency window; -//! `MailboxSoaOwner::try_advance_phase` stays the SOLE lifecycle mutator. +//! - **cast** — one logical thought staged into the open cycle (no WAL write). +//! - **chunk** — a disjoint / row-indexed slice of the cycle's final image. +//! - **cycle** — the 64k-row frame. Its **seal is ONE WAL write** → one version. //! -//! ## The ordering invariant (what the falsifiers prove) +//! Modelling the durable unit as one *cast* (a WAL write per thought) is the +//! anti-pattern this file corrects: it would pay WAL header + durability-wait + +//! version-metadata 64k times. Here the WAL is amortized — many logical thoughts +//! → one durable boundary → one published version. //! -//! - **No durable write ⇒ no receipt ⇒ no step.** A failed append yields no -//! [`DurableReceipt`], and [`apply_durable_step`] cannot be reached without one. -//! - **The step is the PAIRED move** (`receipt.paired_move.to`), never a generic -//! `next_phases().first()`. The move the thought cast rides in the receipt. -//! - A receipt is applied only to **its own** owner ([`PersistError::OwnerMismatch`]), -//! and only if the move itself was minted for that owner (`mv.mailbox == owner` — -//! checked at persist time so a cross-owner move never becomes durable). +//! ## Physical race safety — WRITE-SIDE order, not read-time repair //! -//! ## Crash-durability: the paired move is CO-LOCATED, not in-memory-only +//! 64k thoughts run concurrently and finish in ARBITRARY CPU order. In +//! [`DetachedCycleBatch::freeze`], the casts are stable-ordered by their existing +//! `stream_position` via [`order_cycle_stably`] BEFORE the single WAL append — so +//! the durable image is already canonical. [`WalSink::scan_sealed`] returns rows in +//! that stored order and **never sorts**: completion order is fixed to canonical +//! order at WRITE time, not repaired on every read (which would lose the whole +//! amortization and hand every downstream reader the wrong weave first). The +//! ordering is write-path WAL preparation, so it lives here — NOT in `temporal.rs`, +//! which owns query-time reading. //! -//! The paired transition witness is **durably co-located with the SoA state in -//! the same persistence generation** (operator ruling). The naïve split persists -//! only the payload and keeps the paired move alive solely in the in-memory -//! [`DurableReceipt`]; then *WAL append lands → process dies before -//! [`apply_durable_step`] → the receipt evaporates → the paired move is lost → -//! the KanbanStep never fires*, even though the durable SoA state moved. The gap -//! is silent: storage advanced, lifecycle did not. +//! ## Epistemic race safety — the sealed read horizon //! -//! So [`DurableWrite::append`] takes the [`DurableWitness`] (owner, cast id, -//! cycle, paired move) **alongside** the payload and lands BOTH atomically. The -//! [`DurableReceipt`] then merely *references* that durable material (via its -//! [`DurableCoordinate`]) for the fast in-process apply; it is **not the only -//! copy**. On restart, [`recover_and_apply`] reads the witnesses back -//! ([`DurableWrite::scan_witnesses`]) and re-applies each owner's pending tail. -//! This is a read of what durable storage already holds — **not** a separate -//! ack / confirmation ledger (`E-ACK-ELIMINATED-1`). +//! Every thought in an open cycle reads only the sealed predecessor +//! (`frame.base_version` = `Vn`); the seal publishes `Vn+1` once. An open cycle's +//! output is **never visible as `Vn` input** — [`WalSink::scan_sealed`] excludes +//! any unsealed cycle. So no thought can read a concurrent sibling's in-flight +//! result as though it were sealed history (`read Vn / write Vn+1`). //! -//! ## Replay order + idempotence — the DURABLE coordinate, never `CastId` +//! ## Coalescing — fold ordered updates into owned rows, then chunk //! -//! Two distinct keys, both durable, neither the resettable `CastId` counter: +//! The final cycle image is the ordered per-row fold of the staged updates (a +//! later stream position overwrites the same row); chunks are disjoint / mergeable +//! slices of that final image, NOT successive whole-image replacements. //! -//! - **Replay ORDER = the durable-log position** ([`DurableCoordinate::log_order`], -//! i.e. `seq`). `BatchWriter`'s `CastId` counter **resets to 0 on -//! every restart**, so two witnesses from different writer lifetimes can share a -//! `cast_id` — ordering by it is unstable across crashes. The WAL position is -//! monotonic across the shard's whole life and never resets; `writer_epoch` -//! fences overlapping writers so positions never collide. One owner writes one -//! shard ⇒ a total order over that owner's witnesses, valid across crashes. -//! `cast_id` survives on the witness only as provenance/audit. -//! - **Idempotence = a durable WATERMARK** (`applied_through`), the last durable -//! coordinate whose move is already reflected in the recovered SoA state. -//! **Phase equality is NOT a sound idempotence key**: the Rubicon lifecycle is -//! cyclic (`Planning → CognitiveWork → Evaluation → Plan → Planning`), so after -//! a completed lap the owner is back at `Planning` and a phase-only check would -//! replay the whole lap. [`recover_and_apply`] skips every witness at or below -//! the watermark; above it, the chain must be contiguous (a `from` that does not -//! match the current phase is a genuine gap/corruption, surfaced as -//! [`PersistError::StalePhase`], never silently skipped). **The watermark MUST -//! be persisted with the SoA state** (same generation as the phase it agrees -//! with) or the cyclic ambiguity returns after a second crash — a single -//! per-owner scalar high-water mark, not a per-thought ledger. +//! ## Scope boundary — storage only, no semantic / rung types minted here //! -//! ## The `DurableWrite` seam + the durability type +//! This layer owns storage + race-freedom, nothing else. It does NOT mint (or +//! carry) semantic-witness, ancestry, awareness, branch, rung, or temporal-policy +//! types — those live upstream (the semantic witness substrate) and downstream +//! (`temporal.rs` owns *query-time* reader-horizon / version reading). The +//! write-side ordering here is generic over a caller-supplied canonical key +//! ([`order_cycle_stably`]); the rung a cycle is aligned at is decided by whoever +//! schedules it, not modelled in [`CycleFrame`]. //! -//! A durable append produces a **coordinate** (shard + writer epoch + opaque -//! monotonic `seq`), NOT a base `DatasetVersion` — the dataset version arrives -//! later via MemTable flush + manifest commit. So [`DurableWrite::append`] returns -//! [`DurableCoordinate`], never a version. This is exactly why the coordinate, -//! not `LanceVersion`, is the durability proof: the write is durable *before* any -//! base manifest version attaches, so the latest local state is identified from -//! the co-located witness's durable position, not from a dataset version that does -//! not exist yet. +//! ## What the tests prove — CONTRACT probes, NOT crash/WAL integration (honest) //! -//! The concrete impl (a lance-having crate) wires lance 7.0.0's OFFICIAL MemWAL — -//! preferring the high-level `ShardWriter::put` (`enable_memtable + durable_write`: -//! insert into the queryable MemTable, release the lock, then await WAL durability -//! off-lock) over the raw `WalAppender::append` primitive. It will co-locate the -//! witness and the payload in ONE durable generation. [`DurableWrite::scan_witnesses`] -//! takes a `from` lower bound so recovery reads only the tail after the last -//! applied coordinate, never the whole log. -//! -//! Keeping the seam a trait leaves the ordering core lance-free and `protoc`-free. -//! **This module builds NO concrete `LanceShardSink`** — per operator ruling the -//! durable-witness reshape + `temporal` layer-1 land first; the production sink -//! comes after. -//! -//! ## What the tests prove — CONTRACT probes, NOT crash/WAL integration (be honest) -//! -//! The `#[cfg(test)]` `FakeSink` is an in-process vector, not a WAL. So the tests -//! here are **ordering/recovery CONTRACT probes** — they prove the Rust seam -//! *reconstructs and re-applies a move from a witness once it is durable*, that -//! replay orders by the durable coordinate (not the resettable `CastId`), that -//! recovery is idempotent under a watermark across a cyclic lap, and that a failed -//! append leaves no witness. They do **NOT** prove real durability: there is no -//! real MemWAL, no process restart, no `RecordBatch` atomic co-location of witness -//! + payload, no epoch fencing, no manifest-independent read. **`compile+test green -//! ≠ storage behaviour proven`** (the Ladybug lesson, `CLAUDE.md`). Real -//! crash-durability is demonstrated only by the deferred concrete Lance sink's own -//! integration test; until then this is a *probed contract*, not a durable system. +//! The `#[cfg(test)]` `FakeWalSink` is in-process maps, not a WAL/Lance table. The +//! tests are **storage/race CONTRACT probes**: one WAL write per cycle, write-side +//! order under scrambled completion, per-row coalescing, no unsealed visibility, +//! sealed read horizon, recovery idempotence. They do **NOT** prove real +//! durability (no MemWAL, restart, atomic `RecordBatch`, or manifest). +//! **`compile+test green ≠ storage proven`** (the Ladybug lesson). **This module +//! builds NO concrete Lance sink.** -use lance_graph_contract::collapse_gate::MailboxId; use lance_graph_contract::kanban::{KanbanColumn, KanbanMove, RubiconTransitionError}; +use lance_graph_contract::scheduler::DatasetVersion; use lance_graph_contract::soa_view::MailboxSoaOwner; -use crate::batch_writer::CastId; -use crate::temporal::{local_trajectory_of, LocalCausalRow}; - -/// A durable write that did not land (the WAL append failed / was fenced). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct WriteFailed(pub String); +pub use lance_graph_contract::collapse_gate::MailboxId; -impl std::fmt::Display for WriteFailed { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "durable write did not land: {}", self.0) - } +/// **Write-side stable ordering** — the loom before the WAL. Stable-order +/// concurrently-produced rows by a caller-supplied canonical key BEFORE the single +/// WAL append, so completion order never becomes storage order (physical race) and +/// no read-time repair is ever needed. Generic over the key until the repository +/// proves a canonical key type; equal keys keep arrival order (stable). +/// +/// **Note (the dense-key nuance):** when the key is already a dense slot index into +/// the cycle image, a stable scatter into predetermined positions is cheaper than a +/// general `O(n log n)` sort on the 64k path — that choice belongs beside the cycle +/// batch (here), not in `temporal.rs`. This general sort is the fallback form. +/// +/// This lives with the persistence cycle, NOT in `temporal.rs`: that module owns +/// query-time reader-horizon / version reading; this is write-path WAL preparation. +pub fn order_cycle_stably(rows: &mut [T], key: impl FnMut(&T) -> K) { + rows.sort_by_key(key); } -impl std::error::Error for WriteFailed {} -/// A backend-neutral **durable coordinate** — where the write landed in the -/// durable log. This is NOT a base Lance `DatasetVersion` (that arrives later via -/// MemTable flush + manifest commit); it is the durable-log coordinate that proves -/// durability now. -/// -/// **API-honest by design.** `seq` is an *opaque, backend-defined* monotonic -/// position — NOT specifically a raw WAL entry offset. Lance 7's chosen high-level -/// path (`ShardWriter::put` with `enable_memtable`) returns a **batch-position** -/// result, NOT `WalAppendResult::entry_position`; the raw `WalAppender::append` -/// returns the entry position. Both are monotonic per shard, so the concrete sink -/// maps whichever its API yields into `seq` — this type never claims a WAL offset -/// the selected API does not return. For lance: `shard` = shard `Uuid` (as `u128`), -/// `writer_epoch` = the writer session that fenced the append. +/// A cycle — one globally-aligned 64k-row sweep whose commit is ONE WAL append and +/// ONE [`DatasetVersion`]. A boring scheduling identity (monotonic), not a +/// semantic object. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CycleId(pub u64); + +/// A cycle's storage identity: which cycle, and the sealed predecessor its thoughts +/// read (`Vn` — the epistemic read horizon; the commit publishes its successor). +/// Storage-only: it carries NO rung / projection / branch / semantic tags (those +/// are decided upstream, never minted in this layer). #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct DurableCoordinate { - /// Opaque shard identity (lance: the shard `Uuid` as `u128`). - pub shard: u128, - /// The writer session/epoch that fenced this append. A new writer lifetime - /// takes a higher epoch; fencing guarantees two epochs never share a `seq`. - /// This is the restart discriminator (the ephemeral `CastId` counter is not). - pub writer_epoch: u64, - /// The **opaque backend-defined monotonic durable position** of this append - /// (raw WAL entry position, or `ShardWriter::put`'s batch position — the sink - /// picks). Monotonic across the shard's whole life; it does **not** reset on - /// writer restart. - pub seq: u64, +pub struct CycleFrame { + pub cycle: CycleId, + /// The sealed predecessor every thought in this cycle reads (`Vn`). + pub base_version: DatasetVersion, } -impl DurableCoordinate { - /// The durable-log total-order key for replay + the watermark comparison. - /// - /// `seq` is monotonic across the shard's whole life and never resets (unlike - /// `BatchWriter`'s `CastId` counter), and `writer_epoch` fences overlapping - /// writers so positions never collide. One owner writes one shard, so this is a - /// total order over that owner's witnesses, valid across crashes. +impl CycleFrame { + /// A cycle reading `base_version` (`Vn`) as its sealed predecessor. #[must_use] - pub fn log_order(&self) -> u64 { - self.seq + pub fn new(cycle: CycleId, base_version: DatasetVersion) -> Self { + Self { + cycle, + base_version, + } } } -/// The durable transition witness — CO-LOCATED with the SoA payload in the same -/// persistence generation so it survives a crash (module doc § Crash-durability). -/// -/// Everything needed to re-apply a pending KanbanStep after a restart lives here: -/// WHO (`owner`), the `paired_move` to apply, and the owner-local `cycle`. -/// `cast_id` rides as **provenance/audit only** — it is NOT the replay-order key -/// (it resets across restarts); the durable [`DurableCoordinate`] the sink assigns -/// at append time is, and it is carried by [`LandedWitness`] on the read path. +/// A persistence **landing** record — WHERE a staged thought lands in the cycle. +/// Boring by design: it says where, never why. Semantic causality (ancestry, +/// evidence, revision) stays in the witness substrate — this record has no +/// `basis`. `stream_position` is the caller's EXISTING canonical order key, the +/// sole write-side ordering input. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct DurableWitness { - /// The mailbox this write is on behalf of — the deinterlace grouping key. The - /// `paired_move` (when present) is minted for this same mailbox — checked at - /// persist time, so a cross-owner move never becomes durable. +pub struct SweepSlot { + pub cycle: CycleId, + /// The existing canonical (textual/stream) order key — the write-side + /// deinterlace input. NOT a new coordinate. + pub stream_position: u64, + /// The mailbox this landing is on behalf of. pub owner: MailboxId, - /// The cast this write realises — **provenance only** (resets across restarts; - /// never the replay-order key). The durable coordinate is the order. - pub cast_id: CastId, - /// The owner-local cycle at the time of the cast (audit / trajectory). - pub cycle: u32, - /// The lifecycle move to apply once this generation is durable. `None` when - /// the cast carried no lifecycle intent (a durable no-step). + /// The SoA row this update writes (coalescing folds same-row updates). + pub row: u64, + /// The lifecycle move this thought cast; applied post-SEAL only. `None` = a + /// no-step landing. pub paired_move: Option, + /// The update payload — a descriptor (a `NodeRowPacket` slice in production; + /// bytes here). Never a semantic witness. + pub payload: Vec, } -/// A [`DurableWitness`] read back from the durable log WITH the durable coordinate -/// the sink assigned at append time — the read-path shape [`scan_witnesses`] -/// returns. The coordinate is what orders replay + drives the idempotence -/// watermark (never the resettable `cast_id`), so it implements [`LocalCausalRow`] -/// keyed on [`DurableCoordinate::log_order`]. +/// A landing read back from a SEALED cycle — its slot plus the version its cycle +/// sealed into (shared by every landing of that cycle). Only ever produced for +/// SEALED cycles; returned in the stored (already canonical) order, NEVER re-sorted. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct LandedWitness { - /// Where this witness durably landed — the total-order key across restarts. - pub coordinate: DurableCoordinate, - /// The witness itself. - pub witness: DurableWitness, +pub struct LandedSlot { + /// The version the cycle sealed into (the cheap coarse timeline). + pub version: DatasetVersion, + pub slot: SweepSlot, } -impl LocalCausalRow for LandedWitness { - fn owner(&self) -> MailboxId { - self.witness.owner - } - fn cast_seq(&self) -> u64 { - // The DURABLE log position — monotonic across restarts — NOT `cast_id.0`. - self.coordinate.log_order() - } -} - -/// The async durable-append + replay seam. -/// -/// `&self` (shared — many casts drain concurrently) and **no owner borrow**: the -/// persistence path must not hold the SoA owner across object-store I/O. Both the -/// [`DurableWitness`] and the `payload` are independently owned (never a borrow of -/// live owner state), so the owner stays free while the WAL hums. `async` because -/// lance's WAL is async and the sink runs on the background persistence path -/// (never the hot thinker path); `async_fn_in_trait` is allowed (generic use -/// only, never `dyn`). -#[allow(async_fn_in_trait)] -pub trait DurableWrite { - /// Durably append the `witness` CO-LOCATED with `payload`, atomically, in ONE - /// persistence generation. `Ok(coordinate)` = both landed together (durable - /// now); `Err` = neither did (⇒ no receipt, ⇒ no step). The witness is the - /// crash-durable copy of the paired move — never in-memory-only. - async fn append( - &self, - witness: &DurableWitness, - payload: &[u8], - ) -> Result; +/// A durable write that did not land (the cycle seal failed / was fenced). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WriteFailed(pub String); - /// Replay seam: read back durably-landed witnesses (each with its - /// [`DurableCoordinate`]), in ascending durable-log order, **strictly after** - /// `from` when given (so recovery reads only the tail past the last applied - /// coordinate, never the whole log). Crash recovery scans this, splits it per - /// owner (`temporal` layer-1), and re-applies pending moves in durable order - /// ([`recover_and_apply`]). Reading the SAME co-located material the receipts - /// merely referenced — NOT a separate ledger. - async fn scan_witnesses( - &self, - from: Option, - ) -> Result, WriteFailed>; +impl std::fmt::Display for WriteFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "cycle seal did not land: {}", self.0) + } } +impl std::error::Error for WriteFailed {} -/// A **detached** cast envelope — the thinker's report, carrying its OWN payload -/// (independently owned, never a borrow of live owner state) so persistence runs -/// with the owner free. Mirrors what `BatchWriter::cast(on_behalf, moves, payload)` -/// stages; at drain the descriptor is resolved to `payload` bytes. -pub struct PersistCast { - /// The mailbox this write is on behalf of. - pub owner: MailboxId, - /// The cast this write realises (from `BatchWriter::cast`). Rides into the - /// [`DurableWitness`] as provenance (NOT the replay-order key). - pub cast_id: CastId, - /// The owner-local cycle at cast time — rides into the witness (audit). - pub cycle: u32, - /// The lifecycle move the thought cast with this write (its `to` is applied - /// post-durability). `None` when the cast carries no lifecycle intent. When - /// present, `paired_move.mailbox` MUST equal `owner` (checked in - /// [`persist_cast`]). - pub paired_move: Option, - /// The bytes to persist — an owned/independent buffer (the concrete sink forms - /// one Arrow `RecordBatch` with the witness as a second column, one generation). - pub payload: Vec, +/// A **detached, frozen, already-deinterlaced** cycle image — the loom's finished +/// fabric, ready for the single WAL append. Built by [`DetachedCycleBatch::freeze`] +/// (temporal deinterlace + per-row coalesce) BEFORE [`WalSink::commit_cycle`], so +/// the durable operation is a pure atomic append of an already-coherent frame. +/// Detached = a snapshot, never a borrow of live mutable SoA. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DetachedCycleBatch { + pub frame: CycleFrame, + /// Landings in canonical `stream_position` order (the temporal deinterlace + /// already ran — the WAL never sees worker-completion order). + pub landings: Vec, + /// The coalesced final image: `row -> last payload in stream order`. + pub image: std::collections::BTreeMap>, } -/// Proof the write landed (a [`DurableCoordinate`]) plus the paired move to apply -/// and the owner it belongs to. Produced by [`persist_cast`] on success; consumed -/// by [`apply_durable_step`]. Its mere existence is the "the write landed" fact — -/// there is no separate ack/confirmation ledger (`E-ACK-ELIMINATED-1`). -/// -/// **This receipt REFERENCES durable material; it is not the only copy of the -/// paired move.** The move is co-located in the durable generation the -/// [`coordinate`](Self::coordinate) points at (via the [`DurableWitness`] that -/// [`persist_cast`] appended). If this in-memory receipt is lost to a crash -/// before [`apply_durable_step`] runs, [`recover_and_apply`] reconstructs the -/// move from that durable generation — the KanbanStep is not lost with the -/// process. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DurableReceipt { - /// The mailbox the durable write was on behalf of. - pub owner: MailboxId, - /// The cast this receipt realises — mirrors the co-located witness's cast id. - pub cast_id: CastId, - /// The paired lifecycle move to apply post-durability. A convenience copy of - /// the co-located witness's move (see the type doc — durable, not only here). - pub paired_move: Option, - /// Where the write landed in the durable log — the reference INTO the durable - /// generation that co-locates the witness with the SoA state. - pub coordinate: DurableCoordinate, +impl DetachedCycleBatch { + /// Freeze concurrently-produced casts into the ordered, coalesced cycle image: + /// (1) [`order_cycle_stably`] stable-orders by `stream_position` — + /// completion order never becomes storage order (physical race); (2) fold + /// same-row updates into owned rows (later stream position wins). The result is + /// detached from any live SoA and ready for exactly one WAL append. + #[must_use] + pub fn freeze(frame: CycleFrame, mut casts: Vec) -> Self { + order_cycle_stably(&mut casts, |s| s.stream_position); + let mut image = std::collections::BTreeMap::new(); + for s in &casts { + image.insert(s.row, s.payload.clone()); + } + Self { + frame, + landings: casts, + image, + } + } } -/// Why a persist / step operation produced no lifecycle advance. +/// Why a persist / seal / step operation produced no lifecycle advance. #[derive(Debug, Clone, PartialEq, Eq)] pub enum PersistError { - /// The durable write did not land — **no receipt exists, so no step can be - /// applied** (the ordering invariant's negative half). + /// The cycle seal did not land — nothing published, so no step. Write(WriteFailed), - /// The paired move is not a legal Rubicon edge from the owner's current phase — - /// surfaced, never silently applied. + /// The paired move is not a legal Rubicon edge from the owner's current phase. Illegal(RubiconTransitionError), - /// A receipt (or the move it carries) was applied to the wrong owner — refused - /// (a receipt only advances the mailbox it was minted for; write-on-behalf - /// never crosses owners). Also raised at persist time when - /// `paired_move.mailbox != owner`, so a cross-owner move never becomes durable. + /// A move minted for a different mailbox than the landing's owner — refused so + /// a cross-owner move never becomes durable (write-on-behalf never crosses + /// owners), and again as a recovery corruption guard. OwnerMismatch { - receipt_owner: MailboxId, - applying_to: MailboxId, + move_owner: MailboxId, + landing_owner: MailboxId, }, - /// The paired move's `from` does not match the owner's current phase. On the - /// synchronous single-receipt path ([`apply_durable_step`]) this is a stale / - /// out-of-order apply (concurrent drains can complete a later receipt first) — - /// surfaced so the caller can **RETRY** it once the missing prefix lands. - /// [`apply_durable_step`] borrows the receipt, so it is NOT consumed on this - /// error and stays retryable (recovery only runs on restart — it is not the - /// happy-path backstop for reordered completions). In recovery it means an - /// above-watermark witness does not chain — a genuine gap/corruption. + /// An above-watermark landing's move `from` does not match the owner's current + /// phase — a gap/corruption in the sealed replay chain (phase equality is a + /// corruption guard, NOT the idempotence key — the watermark is). StalePhase { owner_phase: KanbanColumn, move_from: KanbanColumn, @@ -339,187 +230,138 @@ impl std::fmt::Display for PersistError { write!(f, "illegal Rubicon transition {:?} -> {:?}", e.from, e.to) } Self::OwnerMismatch { - receipt_owner, - applying_to, + move_owner, + landing_owner, } => write!( f, - "move for mailbox {receipt_owner} applied to mailbox {applying_to}" + "move for mailbox {move_owner} on a landing owned by {landing_owner}" ), Self::StalePhase { owner_phase, move_from, } => write!( f, - "stale move: owner at {owner_phase:?}, move.from {move_from:?}" + "stale/corrupt replay: owner at {owner_phase:?}, move.from {move_from:?}" ), } } } impl std::error::Error for PersistError {} -/// **Phase 1 — async persistence, NO owner borrow.** Form the crash-durable -/// [`DurableWitness`] and append it CO-LOCATED with the cast's payload in one -/// generation; on success return a [`DurableReceipt`] (which merely references -/// that durable material), on failure a [`PersistError::Write`] and **no -/// receipt**. `O` (the owner) does not appear in this signature at all — it is -/// never borrowed across the WAL / object-store await, so the owner stays free -/// while durability runs. +/// The WAL seam — **one durable append per cycle**. [`commit_cycle`](WalSink::commit_cycle) +/// takes an ALREADY-deinterlaced, ALREADY-frozen [`DetachedCycleBatch`] (the loom +/// ran before the WAL) and performs the single atomic append → one +/// [`DatasetVersion`], visible all-or-nothing. Reads return committed landings +/// only, in the STORED canonical order — this seam NEVER sorts (order is a +/// write-side property, fixed before the append). /// -/// Rejects a `paired_move` minted for a different mailbox -/// ([`PersistError::OwnerMismatch`]) BEFORE the append, so a cross-owner move -/// never becomes durable (the write-on-behalf invariant, enforced at the source). -pub async fn persist_cast( - sink: &W, - cast: PersistCast, -) -> Result { - if let Some(mv) = cast.paired_move { - if mv.mailbox != cast.owner { - return Err(PersistError::OwnerMismatch { - receipt_owner: mv.mailbox, - applying_to: cast.owner, - }); - } - } - let witness = DurableWitness { - owner: cast.owner, - cast_id: cast.cast_id, - cycle: cast.cycle, - paired_move: cast.paired_move, - }; - let coordinate = sink - .append(&witness, &cast.payload) - .await - .map_err(PersistError::Write)?; - Ok(DurableReceipt { - owner: cast.owner, - cast_id: cast.cast_id, - paired_move: cast.paired_move, - coordinate, - }) +/// `&self` (shared) and **no owner borrow**: the persistence path never holds the +/// SoA owner across I/O. +#[allow(async_fn_in_trait)] +pub trait WalSink { + /// The SINGLE amortized WAL append for a whole cycle: commit `batch` (already + /// ordered + coalesced) against the sealed predecessor `base`, publishing + /// exactly one new [`DatasetVersion`] atomically. This is the ONLY durable op — + /// 64k thoughts → one append, not 64k appends. `async` because the concrete WAL + /// append is async. + async fn commit_cycle( + &self, + base: DatasetVersion, + batch: DetachedCycleBatch, + ) -> Result; + + /// Read back COMMITTED landings (never an uncommitted cycle), in the STORED + /// canonical order — this seam does NOT sort. Optionally only cycles after + /// `from_version`. + async fn scan_sealed( + &self, + from_version: Option, + ) -> Result, WriteFailed>; + + /// The CHEAP coarse timeline: `(CycleId, DatasetVersion)` per committed cycle, + /// WITHOUT replaying any landing. The read a downstream time-series consumer + /// (e.g. `stockfish-rs`, another session) does — a lookup of the version table + /// over already-coherent frames. + async fn versions(&self) -> Result, WriteFailed>; } -/// **Phase 2 — synchronous owner-local completion, NO await.** Given a -/// [`DurableReceipt`] (⇒ the write already landed), apply the PAIRED move via the -/// owner's checked `try_advance_phase`. The exclusive `&mut owner` is held only -/// here, outside the storage-latency window. Refuses a receipt minted for a -/// different owner ([`PersistError::OwnerMismatch`]). -/// -/// Returns `Ok(Some(step))` (paired move applied), `Ok(None)` (receipt carried no -/// move — a durable no-step), or an error. The transition target is -/// `receipt.paired_move.to` — never a generic successor. -/// -/// **Reordered receipts stay RETRYABLE — the receipt is BORROWED, not consumed.** -/// `DurableWrite` explicitly supports concurrent drains, so a later append can -/// complete first; applying its receipt while the owner is still at the earlier -/// phase yields [`PersistError::StalePhase`]. Because this takes `&receipt`, the -/// caller still owns it and re-applies it once the missing prefix lands — the -/// KanbanStep is NOT lost on the happy path (recovery only runs on restart, so it -/// is not the backstop for ordinary completion reordering). Per-owner completion -/// order is the caller's to sequence (buffer the non-contiguous tail); cross-owner -/// drains stay fully concurrent. -pub fn apply_durable_step( - owner: &mut O, - receipt: &DurableReceipt, -) -> Result, PersistError> { - if receipt.owner != owner.mailbox_id() { - return Err(PersistError::OwnerMismatch { - receipt_owner: receipt.owner, - applying_to: owner.mailbox_id(), - }); - } - match receipt.paired_move { - Some(mv) => { - // Defense in depth: the move must also be minted for this owner (the - // envelope check above only compares receipt.owner). - if mv.mailbox != owner.mailbox_id() { +/// Deinterlace + freeze a whole cycle's casts and commit it in ONE WAL append. +/// The loom ([`order_cycle_stably`]) and the freeze run in +/// [`DetachedCycleBatch::freeze`] BEFORE the single [`WalSink::commit_cycle`], so +/// completion order never reaches storage. Rejects a cross-owner move (and a +/// cycle-id mismatch) before the batch is built. Returns the one published version. +pub async fn persist_cycle( + sink: &S, + frame: CycleFrame, + casts: Vec, +) -> Result { + for c in &casts { + if c.cycle != frame.cycle { + return Err(PersistError::Write(WriteFailed(format!( + "cast cycle {:?} != frame cycle {:?}", + c.cycle, frame.cycle + )))); + } + if let Some(mv) = c.paired_move { + if mv.mailbox != c.owner { return Err(PersistError::OwnerMismatch { - receipt_owner: mv.mailbox, - applying_to: owner.mailbox_id(), + move_owner: mv.mailbox, + landing_owner: c.owner, }); } - // The move's `from` must match the owner's current phase: on the - // synchronous path a mismatch is a stale / out-of-order apply and is - // surfaced (safe to drop — the durable witness replays it). - if mv.from != owner.phase() { - return Err(PersistError::StalePhase { - owner_phase: owner.phase(), - move_from: mv.from, - }); - } - owner - .try_advance_phase(mv.to) - .map(Some) - .map_err(PersistError::Illegal) } - None => Ok(None), } + // Loom-before-WAL: order + coalesce + detach, THEN one atomic append. + let batch = DetachedCycleBatch::freeze(frame, casts); + sink.commit_cycle(frame.base_version, batch) + .await + .map_err(PersistError::Write) } -/// The result of a [`recover_and_apply`] pass: the moves actually applied, and the -/// new durable **watermark** the caller must persist WITH the recovered SoA state -/// so the next recovery is idempotent (skips everything at or below it). +/// The result of a [`recover_and_apply`] pass: the moves applied this pass, and the +/// new stream-position **watermark** the caller must persist WITH the owner's +/// sealed cycle state so the next recovery is idempotent. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Recovered { - /// The paired moves applied this pass, in durable order. pub applied: Vec, - /// The highest durable coordinate now accounted for (the new watermark), or - /// the input `applied_through` when nothing was applied. Persist it with the - /// owner's phase (same generation) — see the module doc § Replay order. - pub watermark: Option, + /// The highest `stream_position` now accounted for (per owner), or the input + /// watermark when nothing was applied. + pub watermark: Option, } -/// **Crash recovery.** Given landed witnesses read back via -/// [`DurableWrite::scan_witnesses`] (a globally-interleaved stream from every -/// owner), an `owner` reconstructed at its durable phase, and the durable -/// `applied_through` watermark persisted alongside that phase, re-apply the -/// owner's PENDING tail in **durable-log order** and return the applied moves plus -/// the new watermark. -/// -/// - `temporal` layer-1 ([`local_trajectory_of`]) deinterlaces the global stream -/// to this owner's own chain, ordered by the durable [`DurableCoordinate`] (NOT -/// the resettable `cast_id`). -/// - Every witness at or below `applied_through` is **already reflected in the -/// recovered SoA state** and is skipped. This watermark — not phase equality — -/// is the idempotence key: the Rubicon lifecycle is cyclic, so after a completed -/// lap the owner is back at `Planning`, and a phase-only check would replay the -/// whole lap (`E-…-NOT-IN-MEMORY-ONLY-1`). -/// - Above the watermark the chain must be contiguous: a move whose `from` does -/// not match the owner's current phase is a genuine gap/corruption and is -/// surfaced ([`PersistError::StalePhase`]); a matching-`from` move that is not a -/// legal Rubicon edge is [`PersistError::Illegal`]. -/// -/// **On error the owner is left mid-chain** (every earlier move in this pass is -/// already applied). The returned `Recovered` is only produced on full success; -/// re-drive from the persisted watermark after resolving the corruption. +/// **Crash recovery — post-SEAL only.** Given SEALED landings read back via +/// [`WalSink::scan_sealed`] (an unsealed cycle is never here — steps are eligible +/// only after the seal), an `owner` reconstructed at its durable phase, and the +/// durable `applied_through` watermark (`stream_position`) persisted alongside that +/// phase, re-apply the owner's PENDING tail in canonical `stream_position` order. /// -/// This is why the paired move MUST be co-located in the durable generation: the -/// witnesses are the only reason a KanbanStep survives a crash between the WAL -/// append and [`apply_durable_step`]. No witnesses ⇒ nothing to replay ⇒ the step -/// is lost (the gap this reshape closes). +/// The landings arrive already ordered (write-side deinterlace at seal), so no +/// sort is done here. The watermark — not phase equality — is the idempotence key +/// (the Rubicon lifecycle is cyclic). Above the watermark the chain must be +/// contiguous: a non-matching `from` is a gap/corruption ([`PersistError::StalePhase`]). +/// On error the owner is left mid-chain; re-drive from the persisted watermark. pub fn recover_and_apply( owner: &mut O, - landed: &[LandedWitness], - applied_through: Option, + sealed: &[LandedSlot], + applied_through: Option, ) -> Result { - let chain = local_trajectory_of(landed, owner.mailbox_id()); - let hw = applied_through.map(|c| c.log_order()); let mut applied = Vec::new(); let mut watermark = applied_through; - for lw in chain { - // Skip everything at or below the durable watermark — already reflected in - // the recovered SoA state (the cyclic-safe idempotence key). - if hw.is_some_and(|hw| lw.coordinate.log_order() <= hw) { + let me = owner.mailbox_id(); + // Already in canonical stream order from the write side; filter to this owner. + for ls in sealed.iter().filter(|ls| ls.slot.owner == me) { + if applied_through.is_some_and(|hw| ls.slot.stream_position <= hw) { continue; } - match lw.witness.paired_move { - None => { - // A durable no-step still advances the watermark past this - // generation (it is accounted for; nothing to apply). - watermark = Some(lw.coordinate); - } + match ls.slot.paired_move { + None => watermark = Some(ls.slot.stream_position), Some(mv) => { - // Above the watermark the chain MUST be contiguous: a non-matching - // `from` is a gap/corruption, not a benign already-applied move. + if mv.mailbox != owner.mailbox_id() { + return Err(PersistError::OwnerMismatch { + move_owner: mv.mailbox, + landing_owner: owner.mailbox_id(), + }); + } if mv.from != owner.phase() { return Err(PersistError::StalePhase { owner_phase: owner.phase(), @@ -530,7 +372,7 @@ pub fn recover_and_apply( .try_advance_phase(mv.to) .map_err(PersistError::Illegal)?; applied.push(step); - watermark = Some(lw.coordinate); + watermark = Some(ls.slot.stream_position); } } } @@ -540,20 +382,20 @@ pub fn recover_and_apply( #[cfg(test)] mod tests { use super::*; - use lance_graph_contract::collapse_gate::MailboxId as MId; use lance_graph_contract::kanban::{ExecTarget, KanbanColumn}; use lance_graph_contract::soa_view::{MailboxSoaOwner, MailboxSoaView}; - use std::sync::atomic::{AtomicU32, Ordering}; + use std::collections::BTreeMap; + use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; - /// Minimal in-RAM owner (mirrors `kanban_actor::tests::TestBoard`). + // ── Minimal in-RAM owner ──────────────────────────────────────────────────── struct FakeOwner { - id: MId, + id: MailboxId, phase: KanbanColumn, cycle: u32, } impl MailboxSoaView for FakeOwner { - fn mailbox_id(&self) -> MId { + fn mailbox_id(&self) -> MailboxId { self.id } fn n_rows(&self) -> usize { @@ -595,101 +437,124 @@ mod tests { } } } + fn owner(phase: KanbanColumn) -> FakeOwner { + FakeOwner { + id: 42, + phase, + cycle: 5, + } + } - /// A `DurableWrite` whose success/failure and call-count are observable, and - /// which RECORDS every witness it lands (with the durable coordinate it - /// assigns) so [`scan_witnesses`] can replay them — this is what makes the - /// crash-recovery falsifier real: the durable log (`landed`) outlives the - /// in-memory receipts. `Sync` (Mutex + Atomic) so the documented concurrent - /// drain can actually be exercised. - struct FakeSink { + // ── The WAL sink fake ──────────────────────────────────────────────────────── + struct SealedCycle { + frame: CycleFrame, + version: DatasetVersion, + /// The landings AS COMMITTED (already write-side ordered before the append). + landings: Vec, + /// The coalesced final image: row -> last value in stream order. + image: BTreeMap>, + } + struct FakeWalSink { succeed: bool, - calls: AtomicU32, - /// The durable log: (landed witness, payload) pairs, assigned a monotonic - /// `seq` that does NOT reset across simulated writer lifetimes. The payload - /// is STORED (not ignored) so a probe can show witness+payload land - /// together — though an in-process `Vec` cannot prove *atomic* co-location - /// (that is the concrete Lance sink's `RecordBatch`, deferred). - landed: Mutex)>>, + sealed: Mutex>, + next_version: AtomicU64, + /// The number of physical WAL appends — must be ONE per committed cycle. + wal_writes: AtomicU64, } - impl FakeSink { - fn new(succeed: bool) -> Self { + impl FakeWalSink { + fn new() -> Self { + Self { + succeed: true, + sealed: Mutex::new(Vec::new()), + next_version: AtomicU64::new(1), + wal_writes: AtomicU64::new(0), + } + } + fn failing() -> Self { Self { - succeed, - calls: AtomicU32::new(0), - landed: Mutex::new(Vec::new()), + succeed: false, + ..Self::new() } } - fn calls(&self) -> u32 { - self.calls.load(Ordering::SeqCst) + fn wal_writes(&self) -> u64 { + self.wal_writes.load(Ordering::SeqCst) } - /// Test-only: the (coordinate.seq, payload) pairs the sink holds — used to - /// show the payload landed alongside its witness, not dropped. - fn landed_payloads(&self) -> Vec<(u64, Vec)> { - self.landed + fn version_count(&self) -> usize { + self.sealed.lock().unwrap().len() + } + fn image_of(&self, cycle: CycleId) -> Option>> { + self.sealed .lock() .unwrap() .iter() - .map(|(lw, p)| (lw.coordinate.seq, p.clone())) - .collect() + .find(|s| s.frame.cycle == cycle) + .map(|s| s.image.clone()) + } + /// Test-only: inject a committed cycle whose landings are DELIBERATELY out + /// of order, to prove `scan_sealed` does not re-sort (order is a write-side + /// property, fixed before the append — never a read-time repair). + fn inject_unordered_committed(&self, frame: CycleFrame, landings: Vec) { + let v = DatasetVersion(self.next_version.fetch_add(1, Ordering::SeqCst)); + self.sealed.lock().unwrap().push(SealedCycle { + frame, + version: v, + landings, + image: BTreeMap::new(), + }); } } - impl DurableWrite for FakeSink { - async fn append( + impl WalSink for FakeWalSink { + async fn commit_cycle( &self, - witness: &DurableWitness, - payload: &[u8], - ) -> Result { - self.calls.fetch_add(1, Ordering::SeqCst); + _base: DatasetVersion, + batch: DetachedCycleBatch, + ) -> Result { if !self.succeed { - // The negative half: a fenced WAL lands NOTHING — no witness, no - // payload — so nothing to replay, so no move and no step. - return Err(WriteFailed("wal fenced".into())); + return Err(WriteFailed("cycle fenced".into())); } - let mut log = self.landed.lock().unwrap(); - // Durable position: monotonic over the log's whole life, 1-based. - let coordinate = DurableCoordinate { - shard: 0xABCD, - writer_epoch: 1, - seq: log.len() as u64 + 1, - }; - log.push(( - LandedWitness { - coordinate, - witness: witness.clone(), - }, - payload.to_vec(), - )); - Ok(coordinate) + // The batch arrives ALREADY deinterlaced + coalesced (loom before WAL). + // THE single amortized WAL append for the whole cycle. + self.wal_writes.fetch_add(1, Ordering::SeqCst); + let version = DatasetVersion(self.next_version.fetch_add(1, Ordering::SeqCst)); + self.sealed.lock().unwrap().push(SealedCycle { + frame: batch.frame, + version, + landings: batch.landings, + image: batch.image, + }); + Ok(version) } - async fn scan_witnesses( + async fn scan_sealed( &self, - from: Option, - ) -> Result, WriteFailed> { - let lb = from.map(|c| c.log_order()); + from_version: Option, + ) -> Result, WriteFailed> { + // Returned in STORED order — no sort. (The order was fixed at seal.) Ok(self - .landed + .sealed .lock() .unwrap() .iter() - .map(|(lw, _payload)| lw) - .filter(|lw| lb.is_none_or(|lb| lw.coordinate.log_order() > lb)) - .cloned() + .filter(|s| from_version.is_none_or(|f| s.version > f)) + .flat_map(|s| { + s.landings.iter().map(|slot| LandedSlot { + version: s.version, + slot: slot.clone(), + }) + }) .collect()) } - } - - fn owner(phase: KanbanColumn) -> FakeOwner { - owner_id(42, phase) - } - fn owner_id(id: MId, phase: KanbanColumn) -> FakeOwner { - FakeOwner { - id, - phase, - cycle: 5, + async fn versions(&self) -> Result, WriteFailed> { + Ok(self + .sealed + .lock() + .unwrap() + .iter() + .map(|s| (s.frame.cycle, s.version)) + .collect()) } } - fn mv(owner: MId, from: KanbanColumn, to: KanbanColumn) -> KanbanMove { + + fn mv(owner: MailboxId, from: KanbanColumn, to: KanbanColumn) -> KanbanMove { KanbanMove { mailbox: owner, from, @@ -698,444 +563,312 @@ mod tests { exec: ExecTarget::Elixir, } } - fn cast(paired_to: Option) -> PersistCast { - cast_of(42, 0, KanbanColumn::Planning, paired_to) - } - fn cast_of( - owner: MId, - cast_id: u64, - from: KanbanColumn, - paired_to: Option, - ) -> PersistCast { - PersistCast { + /// A landing for `owner` at `stream_position`, moving `from→to`, writing `row`. + fn slot( + owner: MailboxId, + cycle: u64, + stream_position: u64, + row: u64, + m: Option<(KanbanColumn, KanbanColumn)>, + ) -> SweepSlot { + SweepSlot { + cycle: CycleId(cycle), + stream_position, owner, - cast_id: CastId(cast_id), - cycle: 0, - paired_move: paired_to.map(|to| mv(owner, from, to)), - payload: vec![cast_id as u8], - } - } - fn receipt_from(from: KanbanColumn, paired_to: Option) -> DurableReceipt { - DurableReceipt { - owner: 42, - cast_id: CastId(0), - paired_move: paired_to.map(|to| mv(42, from, to)), - coordinate: DurableCoordinate { - shard: 0xABCD, - writer_epoch: 1, - seq: 7, - }, + row, + paired_move: m.map(|(f, t)| mv(owner, f, t)), + payload: vec![stream_position as u8], } } - fn receipt_for(paired_to: Option) -> DurableReceipt { - receipt_from(KanbanColumn::Planning, paired_to) - } - - // ── Phase 1: async persistence, NO owner borrow ──────────────────────────── - - #[tokio::test] - async fn a_successful_write_yields_a_receipt_carrying_the_paired_move() { - let sink = FakeSink::new(true); - let r = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))) - .await - .expect("write landed"); - assert_eq!(r.owner, 42); - assert_eq!( - r.paired_move.map(|m| m.to), - Some(KanbanColumn::CognitiveWork) - ); - assert_eq!( - r.coordinate.seq, 1, - "the durable coordinate rides (first witness landed)" - ); - assert_eq!(r.cast_id, CastId(0), "the receipt mirrors the cast id"); - assert_eq!(sink.calls(), 1, "the write was attempted once"); - // The witness LANDED durably (co-located) — not only in the receipt. - let scanned = sink.scan_witnesses(None).await.expect("scan"); - assert_eq!(scanned.len(), 1, "one witness durably co-located"); - assert_eq!( - scanned[0].witness.paired_move.map(|m| m.to), - Some(KanbanColumn::CognitiveWork), - "the paired move is in the DURABLE witness, not only the in-memory receipt", - ); - // …and the PAYLOAD landed with it at the same seq (not ignored). (An - // in-process Vec cannot prove *atomic* co-location — that is the concrete - // Lance sink's RecordBatch; the module doc states this honestly.) - assert_eq!( - sink.landed_payloads(), - vec![(1u64, vec![0u8])], - "the payload landed alongside the witness at seq 1", - ); - } - - #[tokio::test] - async fn a_failed_write_yields_no_receipt() { - let sink = FakeSink::new(false); - let r = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; - assert_eq!( - r, - Err(PersistError::Write(WriteFailed("wal fenced".into()))) - ); - assert_eq!(sink.calls(), 1, "the write was attempted"); - } + // ── FALSIFIER (headline): one WAL write per cycle — the amortization ───────── #[tokio::test] - async fn a_fenced_write_lands_no_witness_so_nothing_can_be_replayed() { - // Negative half, at the durable layer: a failed append records NO witness, - // so a subsequent crash-recovery scan finds nothing to replay ⇒ no move. - let sink = FakeSink::new(false); - let _ = persist_cast(&sink, cast(Some(KanbanColumn::CognitiveWork))).await; + async fn a_whole_cycle_of_casts_is_one_wal_write_one_version() { + let sink = FakeWalSink::new(); + let frame = CycleFrame::new(CycleId(1), DatasetVersion(0)); + // 100 concurrent thoughts stage into an owned cast vector — building it + // touches NO WAL (staging is the caller's, not the sink's). + let casts: Vec = (0..100u64).map(|i| slot(42, 1, i, i, None)).collect(); assert_eq!( - sink.scan_witnesses(None).await.expect("scan").len(), + sink.wal_writes(), 0, - "a fenced WAL leaves no durable witness — no move, no step", + "staging writes no WAL — pure amortization" ); + assert_eq!(sink.version_count(), 0, "no version before the seal"); + // persist_cycle is the SINGLE amortized WAL write for the whole cycle. + let version = persist_cycle(&sink, frame, casts).await.unwrap(); + assert_eq!(sink.wal_writes(), 1, "100 casts → exactly ONE WAL write"); + assert_eq!(version, DatasetVersion(1), "→ exactly one version"); + assert_eq!(sink.version_count(), 1); } + // ── FALSIFIER: write-side order under scrambled completion (physical race) ─── #[tokio::test] - async fn a_cross_owner_paired_move_is_rejected_before_it_becomes_durable() { - // A move minted for mailbox 99 riding in a cast on behalf of 42 must be - // refused at persist time — a cross-owner move never lands durably. - let sink = FakeSink::new(true); - let cast = PersistCast { - owner: 42, - cast_id: CastId(0), - cycle: 0, - paired_move: Some(mv(99, KanbanColumn::Planning, KanbanColumn::CognitiveWork)), - payload: vec![], - }; - assert_eq!( - persist_cast(&sink, cast).await, - Err(PersistError::OwnerMismatch { - receipt_owner: 99, - applying_to: 42 - }), - ); - assert_eq!(sink.calls(), 0, "the write was never attempted"); + async fn scrambled_completion_lands_in_canonical_stream_order_at_write_time() { + let sink = FakeWalSink::new(); + // Thoughts "finish" (stage) in scrambled CPU order: stream 2, 0, 3, 1. + let sealed = persist_cycle( + &sink, + CycleFrame::new(CycleId(5), DatasetVersion(0)), + vec![ + slot(42, 5, 2, 20, None), + slot(42, 5, 0, 10, None), + slot(42, 5, 3, 30, None), + slot(42, 5, 1, 11, None), + ], + ) + .await + .unwrap(); + // scan_sealed does NOT sort; the order came from the WRITE side (seal). + let order: Vec = sink + .scan_sealed(None) + .await + .unwrap() + .iter() + .filter(|l| l.version == sealed) + .map(|l| l.slot.stream_position) + .collect(); assert_eq!( - sink.scan_witnesses(None).await.expect("scan").len(), - 0, - "nothing durable", + order, + vec![0, 1, 2, 3], + "canonical stream order was fixed at WRITE time, not repaired on read", ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn concurrent_drains_both_land_durably() { - // The DurableWrite doc claims &self is shared because many casts drain - // concurrently. Exercise it: two persist_cast futures on a shared sink. - let sink = std::sync::Arc::new(FakeSink::new(true)); - let s1 = sink.clone(); - let s2 = sink.clone(); - let (a, b) = tokio::join!( - async move { - persist_cast( - &*s1, - cast_of( - 42, - 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), - ), - ) - .await - }, - async move { - persist_cast( - &*s2, - cast_of( - 43, - 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), - ), - ) - .await - }, + /// The complement: `scan_sealed` itself performs NO sort — order is purely a + /// write-side property. Inject a deliberately out-of-order sealed cycle and + /// prove scan returns it AS-STORED (a read-time sort would "fix" it). + #[tokio::test] + async fn scan_sealed_does_not_repair_order_on_read() { + let sink = FakeWalSink::new(); + sink.inject_unordered_committed( + CycleFrame::new(CycleId(9), DatasetVersion(0)), + vec![ + slot(42, 9, 3, 30, None), + slot(42, 9, 1, 10, None), + slot(42, 9, 2, 20, None), + ], ); - assert!(a.is_ok() && b.is_ok(), "both drains landed"); - assert_eq!(sink.calls(), 2); + let order: Vec = sink + .scan_sealed(None) + .await + .unwrap() + .iter() + .map(|l| l.slot.stream_position) + .collect(); assert_eq!( - sink.scan_witnesses(None).await.expect("scan").len(), - 2, - "both witnesses durably recorded under concurrent drain", + order, + vec![3, 1, 2], + "scan_sealed returns stored order verbatim — it never sorts (order is write-side)", ); } - // ── Phase 2: synchronous owner-local completion, NO await ─────────────────── - - #[test] - fn applying_a_receipt_advances_the_owner_by_the_paired_move() { - let mut o = owner(KanbanColumn::Planning); - let step = apply_durable_step(&mut o, &receipt_for(Some(KanbanColumn::CognitiveWork))) - .expect("legal") - .expect("a paired move"); - assert_eq!(step.to, KanbanColumn::CognitiveWork); - assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "advanced once"); - assert_eq!(o.current_cycle(), 6); - } - - #[test] - fn applying_a_receipt_uses_the_paired_move_not_the_generic_successor() { - // THE key falsifier. From `Evaluation` the generic forward arc is `Commit`; - // the receipt carries the free-won't veto `Evaluation → Prune`. The step - // must be the PAIRED move (Prune), never the generic successor (Commit). - assert_eq!( - KanbanColumn::Evaluation.next_phases().first(), - Some(&KanbanColumn::Commit), - "precondition: generic successor is Commit", - ); - let mut o = owner(KanbanColumn::Evaluation); - let step = apply_durable_step( - &mut o, - &receipt_from(KanbanColumn::Evaluation, Some(KanbanColumn::Prune)), + // ── FALSIFIER: per-row coalescing (not last-chunk-wins) ────────────────────── + #[tokio::test] + async fn same_row_updates_coalesce_distinct_rows_survive() { + let sink = FakeWalSink::new(); + persist_cycle( + &sink, + CycleFrame::new(CycleId(3), DatasetVersion(0)), + vec![ + // Two updates to ROW 7 (stream 0 then 2) — coalesce to the later. + SweepSlot { + payload: vec![0xA0], + ..slot(42, 3, 0, 7, None) + }, + // A different ROW 8 — survives independently. + SweepSlot { + payload: vec![0xB0], + ..slot(42, 3, 1, 8, None) + }, + SweepSlot { + payload: vec![0xA1], + ..slot(42, 3, 2, 7, None) + }, + ], ) - .expect("legal") - .expect("paired veto"); - assert_eq!(step.to, KanbanColumn::Prune, "the paired veto, not Commit"); - assert_eq!(o.phase(), KanbanColumn::Prune); - assert_ne!( - o.phase(), - KanbanColumn::Commit, - "generic successor NOT taken" - ); - } - - #[test] - fn a_receipt_with_no_paired_move_is_a_durable_no_step() { - let mut o = owner(KanbanColumn::CognitiveWork); - let r = apply_durable_step(&mut o, &receipt_for(None)); - assert_eq!(r, Ok(None), "durable, but no step"); - assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "phase unchanged"); - } - - #[test] - fn an_illegal_paired_edge_is_surfaced_not_applied() { - // Planning → Evaluation skips CognitiveWork — illegal. Surfaced; owner - // untouched (the checked airgap holds in the owner-local phase). - let mut o = owner(KanbanColumn::Planning); - let r = apply_durable_step(&mut o, &receipt_for(Some(KanbanColumn::Evaluation))); - assert!(matches!(r, Err(PersistError::Illegal(_)))); - assert_eq!(o.phase(), KanbanColumn::Planning, "owner untouched"); - } - - #[test] - fn a_receipt_is_refused_for_a_foreign_owner() { - // A receipt minted for mailbox 42 must never advance mailbox 99. - let mut other = owner_id(99, KanbanColumn::Planning); - let r = apply_durable_step(&mut other, &receipt_for(Some(KanbanColumn::CognitiveWork))); - assert!(matches!( - r, - Err(PersistError::OwnerMismatch { - receipt_owner: 42, - applying_to: 99 - }) - )); + .await + .unwrap(); + let image = sink.image_of(CycleId(3)).unwrap(); + assert_eq!(image.len(), 2, "two distinct rows in the final image"); assert_eq!( - other.phase(), - KanbanColumn::Planning, - "foreign owner untouched" + image.get(&7), + Some(&vec![0xA1]), + "row 7 coalesced to the later update" ); + assert_eq!(image.get(&8), Some(&vec![0xB0]), "row 8 survived, disjoint"); } - #[test] - fn a_stale_move_on_the_sync_path_is_surfaced_not_silently_reapplied() { - // The sync path surfaces a stale/out-of-order move LOUDLY. A `Planning→…` - // move applied to an owner already at `CognitiveWork` is refused, owner - // untouched — and the borrowed receipt stays usable (see the retry test). - let mut o = owner(KanbanColumn::CognitiveWork); - let r = apply_durable_step(&mut o, &receipt_for(Some(KanbanColumn::CognitiveWork))); + // ── FALSIFIER: no partial visibility — an unsealed cycle is invisible ──────── + #[tokio::test] + async fn an_unsealed_cycle_is_invisible_epistemic_horizon() { + let sink = FakeWalSink::new(); + // A cycle's casts are staged in an owned vector held by the caller — NOT in + // the sink. Until the single atomic commit_cycle runs, nothing is visible: + // an open cycle's output is NOT readable as Vn input (read Vn / write Vn+1). + let casts = vec![slot( + 42, + 9, + 0, + 0, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + )]; assert_eq!( - r, - Err(PersistError::StalePhase { - owner_phase: KanbanColumn::CognitiveWork, - move_from: KanbanColumn::Planning, - }), + sink.scan_sealed(None).await.unwrap().len(), + 0, + "before the seal, no landing is visible", ); - assert_eq!(o.phase(), KanbanColumn::CognitiveWork, "owner untouched"); - } - - /// FALSIFIER (Codex/ChatGPT): concurrent drains can complete a LATER receipt - /// before its predecessor. The later receipt must NOT be lost — `apply_durable_step` - /// borrows it, so on `StalePhase` the caller still owns it and retries once the - /// missing prefix lands. Both steps fire, in order, on the HAPPY path (no crash). - #[test] - fn a_reordered_receipt_is_retryable_not_lost() { - // Two casts for one owner: r8 (Planning→CognitiveWork) then - // r9 (CognitiveWork→Evaluation). r9's append completes FIRST. - let r8 = receipt_from(KanbanColumn::Planning, Some(KanbanColumn::CognitiveWork)); - let r9 = receipt_from(KanbanColumn::CognitiveWork, Some(KanbanColumn::Evaluation)); - let mut o = owner(KanbanColumn::Planning); - - // r9 arrives first — stale (owner still at Planning). Surfaced, NOT consumed. - assert!(matches!( - apply_durable_step(&mut o, &r9), - Err(PersistError::StalePhase { .. }), - )); - assert_eq!( - o.phase(), - KanbanColumn::Planning, - "r9 did not apply out of order" + assert!( + sink.versions().await.unwrap().is_empty(), + "no version before seal" ); - - // r8's append completes; it applies. - apply_durable_step(&mut o, &r8) - .expect("r8 legal") - .expect("moved"); - assert_eq!(o.phase(), KanbanColumn::CognitiveWork); - - // The caller RETRIES the still-owned r9 — it now applies. Nothing was lost, - // and no crash/recovery was needed (the happy path stays correct). - apply_durable_step(&mut o, &r9) - .expect("r9 legal now") - .expect("moved"); + // The seal is all-or-nothing: the whole cycle appears at once, never partially. + persist_cycle(&sink, CycleFrame::new(CycleId(9), DatasetVersion(0)), casts) + .await + .unwrap(); assert_eq!( - o.phase(), - KanbanColumn::Evaluation, - "both steps fired, in order" + sink.scan_sealed(None).await.unwrap().len(), + 1, + "visible only after seal" ); } - // ── Crash recovery: the co-located witness replays after the receipt is lost ─ - - /// Persist a chain of casts through the sink (they land durably) and return the - /// scanned landed witnesses — the "durable log" a restart reads back. - async fn persist_and_scan(sink: &FakeSink, casts: Vec) -> Vec { - for c in casts { - persist_cast(sink, c).await.expect("landed"); - } - sink.scan_witnesses(None).await.expect("scan") - } - - /// THE crash falsifier (operator): WAL append succeeds, then the process dies - /// BEFORE the sync step — every in-memory [`DurableReceipt`] is dropped. On - /// restart the owner is reconstructed at its durable phase (+ its durable - /// watermark) and [`recover_and_apply`] replays the PAIRED move from the - /// co-located witness. Idempotent: a second recovery from the returned - /// watermark applies nothing. + // ── FALSIFIER: sealed read horizon — a cycle reads the sealed predecessor ──── #[tokio::test] - async fn a_crash_after_durable_write_replays_the_move_from_the_witness() { - let sink = FakeSink::new(true); - let receipt = persist_cast( + async fn a_cycle_reads_the_sealed_predecessor_not_an_in_flight_sibling() { + let sink = FakeWalSink::new(); + // Cycle 1 seals → V1. + let s1 = persist_cycle( &sink, - cast_of( - 42, - 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), - ), + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![slot(42, 1, 0, 0, None)], ) .await - .expect("write landed"); - drop(receipt); // ── CRASH ── never called apply_durable_step. - - let landed = sink.scan_witnesses(None).await.expect("scan"); - let mut o = owner(KanbanColumn::Planning); // durable phase; no watermark yet - let rec = recover_and_apply(&mut o, &landed, None).expect("recovery legal"); + .unwrap(); + // Cycle 2's base_version is the SEALED V1 — never an in-flight value. + let f2 = CycleFrame::new(CycleId(2), s1); assert_eq!( - rec.applied.iter().map(|m| m.to).collect::>(), - vec![KanbanColumn::CognitiveWork], - "the pending move was reconstructed from the durable witness and applied", + f2.base_version, s1, + "the next cycle reads the SEALED predecessor version (Vn), not in-flight state", ); + let s2 = persist_cycle(&sink, f2, vec![slot(99, 2, 0, 0, None)]) + .await + .unwrap(); + assert_eq!(s2, DatasetVersion(2), "and publishes exactly its own Vn+1"); + } + + // ── FALSIFIER: cheap time-series — version table only, no landing replay ───── + #[tokio::test] + async fn downstream_time_series_reads_the_version_table_not_the_landings() { + let sink = FakeWalSink::new(); + for c in 1..=3u64 { + persist_cycle( + &sink, + CycleFrame::new(CycleId(c), DatasetVersion(c - 1)), + vec![slot(42, c, 0, 0, None)], + ) + .await + .unwrap(); + } assert_eq!( - o.phase(), - KanbanColumn::CognitiveWork, - "the step fired on recovery" + sink.versions().await.unwrap(), + vec![ + (CycleId(1), DatasetVersion(1)), + (CycleId(2), DatasetVersion(2)), + (CycleId(3), DatasetVersion(3)), + ], + "history scales with CYCLES; a time-series consumer just looks up the version table", ); - assert!(rec.watermark.is_some(), "a new watermark to persist"); - - // Idempotent: recovery re-run FROM THE WATERMARK applies nothing. - let again = recover_and_apply(&mut o, &landed, rec.watermark).expect("idempotent"); - assert!(again.applied.is_empty(), "second recovery is a no-op"); - assert_eq!(o.phase(), KanbanColumn::CognitiveWork); } - /// FALSIFIER (operator): one owner's durable batch replays in DURABLE order, - /// and the globally-interleaved OTHER owner's witnesses are deinterlaced away - /// (temporal layer-1). A durable no-step interleaved into the chain is skipped - /// while the following move still applies. + // ── Recovery over sealed landings (post-seal only) ─────────────────────────── #[tokio::test] - async fn recovery_replays_one_owners_batch_in_order_skipping_no_steps_and_other_owners() { - let sink = FakeSink::new(true); - // Durable-log order interleaves two owners + a no-step for 42: - // c0: 42 Planning→CognitiveWork - // c1: 99 Planning→CognitiveWork (other owner, interleaved) - // c2: 42 no-step (paired_move None) - // c3: 42 CognitiveWork→Evaluation - let landed = persist_and_scan( + async fn recovery_replays_an_owners_chain_in_stream_order_skipping_others() { + let sink = FakeWalSink::new(); + persist_cycle( &sink, + CycleFrame::new(CycleId(4), DatasetVersion(0)), vec![ - cast_of( + slot( 42, + 4, 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), + 0, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), ), - cast_of( + slot( 99, - 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), + 4, + 1, + 1, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), ), - cast_of(42, 1, KanbanColumn::Planning, None), - cast_of( + slot( 42, + 4, 2, - KanbanColumn::CognitiveWork, - Some(KanbanColumn::Evaluation), + 2, + Some((KanbanColumn::CognitiveWork, KanbanColumn::Evaluation)), ), ], ) - .await; - assert_eq!(landed.len(), 4, "all four witnesses are durable"); - - let mut o = owner(KanbanColumn::Planning); // owner 42 at its durable phase - let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); + .await + .unwrap(); + let sealed = sink.scan_sealed(None).await.unwrap(); + let mut o = owner(KanbanColumn::Planning); + let rec = recover_and_apply(&mut o, &sealed, None).unwrap(); assert_eq!( rec.applied.iter().map(|m| m.to).collect::>(), vec![KanbanColumn::CognitiveWork, KanbanColumn::Evaluation], - "owner 42's OWN chain, in durable order — 99's cast + the no-step skipped", - ); - assert_eq!( - o.phase(), - KanbanColumn::Evaluation, - "advanced two local steps" + "owner 42's chain in stream order — owner 99's interleaved landing skipped", ); + assert_eq!(o.phase(), KanbanColumn::Evaluation); } - /// THE cyclic idempotence falsifier (Codex/CodeRabbit Critical). A full lap - /// `Planning → CognitiveWork → Evaluation → Plan → Planning` leaves the owner - /// back at `Planning`. Phase equality alone would replay the whole lap; the - /// durable WATERMARK makes the second recovery a no-op. The negative control - /// (recovering WITHOUT the persisted watermark) proves the watermark is - /// load-bearing by reproducing the double-lap. #[tokio::test] - async fn cyclic_recovery_is_idempotent_only_with_the_durable_watermark() { - // Precondition: the lap is a real cycle in the Rubicon graph. + async fn cyclic_recovery_is_idempotent_only_with_the_watermark() { assert_eq!(KanbanColumn::Plan.next_phases(), &[KanbanColumn::Planning]); - let sink = FakeSink::new(true); - let landed = persist_and_scan( + let sink = FakeWalSink::new(); + persist_cycle( &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), vec![ - cast_of( + slot( 42, + 1, + 0, 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), ), - cast_of( + slot( 42, 1, - KanbanColumn::CognitiveWork, - Some(KanbanColumn::Evaluation), + 1, + 1, + Some((KanbanColumn::CognitiveWork, KanbanColumn::Evaluation)), + ), + slot( + 42, + 1, + 2, + 2, + Some((KanbanColumn::Evaluation, KanbanColumn::Plan)), + ), + slot( + 42, + 1, + 3, + 3, + Some((KanbanColumn::Plan, KanbanColumn::Planning)), ), - cast_of(42, 2, KanbanColumn::Evaluation, Some(KanbanColumn::Plan)), - cast_of(42, 3, KanbanColumn::Plan, Some(KanbanColumn::Planning)), ], ) - .await; - + .await + .unwrap(); + let sealed = sink.scan_sealed(None).await.unwrap(); let mut o = owner(KanbanColumn::Planning); - let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); + + let rec = recover_and_apply(&mut o, &sealed, None).unwrap(); assert_eq!(rec.applied.len(), 4, "the whole lap applied once"); assert_eq!( o.phase(), @@ -1143,144 +876,77 @@ mod tests { "back at Planning after a lap" ); - // WITH the persisted watermark: second recovery skips the entire lap. - let good = recover_and_apply(&mut o, &landed, rec.watermark).expect("legal"); + let good = recover_and_apply(&mut o, &sealed, rec.watermark).unwrap(); assert!( good.applied.is_empty(), - "watermark makes cyclic recovery idempotent", + "watermark makes cyclic recovery idempotent" ); - assert_eq!(o.phase(), KanbanColumn::Planning); - // NEGATIVE CONTROL: WITHOUT the watermark (as if it were not persisted), - // phase equality replays the whole lap — the bug the watermark fixes. - let bug = recover_and_apply(&mut o, &landed, None).expect("legal"); + let bug = recover_and_apply(&mut o, &sealed, None).unwrap(); assert_eq!( bug.applied.len(), 4, - "without the durable watermark the cyclic lap is replayed — watermark is load-bearing", + "without the watermark the cyclic lap replays" ); } - /// FALSIFIER (Bugbot High): `CastId` resets across writer restarts, so two - /// lifetimes can share `cast_id` 0. Replay MUST order by the durable WAL - /// position (which does not reset), not `cast_id`. Two lifetimes each cast - /// `cast_id 0`, at distinct WAL positions, forming one owner's chain — recovery - /// applies them in durable order regardless of the colliding cast ids. #[tokio::test] - async fn recovery_orders_by_durable_position_not_the_resettable_cast_id() { - let sink = FakeSink::new(true); - // Lifetime 1 (cast_id 0) then lifetime 2 (cast_id 0 again) — the counter - // reset. Distinct WAL positions (1, 2) come from the durable log. - let landed = persist_and_scan( - &sink, - vec![ - cast_of( - 42, - 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), - ), - cast_of( - 42, - 0, - KanbanColumn::CognitiveWork, - Some(KanbanColumn::Evaluation), - ), - ], - ) - .await; - assert_eq!( - landed - .iter() - .map(|l| l.witness.cast_id.0) - .collect::>(), - vec![0, 0], - "the cast ids collide (counter reset across lifetimes)", - ); - assert_eq!( - landed.iter().map(|l| l.coordinate.seq).collect::>(), - vec![1, 2], - "but the durable WAL positions are distinct + monotonic", - ); - - let mut o = owner(KanbanColumn::Planning); - let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); - assert_eq!( - rec.applied.iter().map(|m| m.to).collect::>(), - vec![KanbanColumn::CognitiveWork, KanbanColumn::Evaluation], - "ordered by durable position despite the colliding cast ids", - ); - assert_eq!(o.phase(), KanbanColumn::Evaluation); + async fn a_cross_owner_move_is_rejected_before_the_cycle_persists() { + let bad = SweepSlot { + paired_move: Some(mv(99, KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + ..slot(42, 1, 0, 0, None) + }; + let sink = FakeWalSink::new(); + assert!(matches!( + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![bad] + ) + .await, + Err(PersistError::OwnerMismatch { + move_owner: 99, + landing_owner: 42 + }), + )); + assert_eq!(sink.wal_writes(), 0, "nothing written on a bad cast"); + assert_eq!(sink.version_count(), 0); } - /// FALSIFIER 5 (operator): the durable proof is the coordinate, NOT a - /// `LanceVersion`. A witness is replayable the instant it lands — before any - /// base manifest version attaches — so recovery identifies the latest LOCAL - /// state from the co-located witness's durable position, not a dataset version. #[tokio::test] - async fn recovery_uses_durable_position_not_a_dataset_version() { - let sink = FakeSink::new(true); - let receipt = persist_cast( + async fn a_fenced_cycle_writes_nothing() { + let sink = FakeWalSink::failing(); + let r = persist_cycle( &sink, - cast_of( + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![slot( 42, + 1, 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), - ), + 0, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + )], ) - .await - .expect("landed"); - // The durability proof carries no dataset version — only a WAL/LSM - // coordinate. (The type has no `DatasetVersion` field to read.) - assert!( - receipt.coordinate.log_order() > 0, - "durable via the WAL coordinate, before any manifest version", - ); - let landed = sink.scan_witnesses(None).await.expect("scan"); - let mut o = owner(KanbanColumn::Planning); - let rec = recover_and_apply(&mut o, &landed, None).expect("legal"); - assert_eq!( - rec.applied.len(), - 1, - "latest local state found from the durable position alone" - ); + .await; + assert!(matches!(r, Err(PersistError::Write(_)))); + assert_eq!(sink.wal_writes(), 0, "no WAL write, no version, no step"); + assert_eq!(sink.scan_sealed(None).await.unwrap().len(), 0); } - /// The bounded scan seam: `scan_witnesses(from)` returns only the tail past a - /// coordinate — recovery need not read the whole log. - #[tokio::test] - async fn scan_from_a_coordinate_returns_only_the_tail() { - let sink = FakeSink::new(true); - let _ = persist_and_scan( - &sink, - vec![ - cast_of( - 42, - 0, - KanbanColumn::Planning, - Some(KanbanColumn::CognitiveWork), - ), - cast_of( - 42, - 1, - KanbanColumn::CognitiveWork, - Some(KanbanColumn::Evaluation), - ), - cast_of(42, 2, KanbanColumn::Evaluation, Some(KanbanColumn::Commit)), - ], - ) - .await; - let after_first = DurableCoordinate { - shard: 0xABCD, - writer_epoch: 1, - seq: 1, - }; - let tail = sink.scan_witnesses(Some(after_first)).await.expect("scan"); - assert_eq!( - tail.iter().map(|l| l.coordinate.seq).collect::>(), - vec![2, 3], - "only witnesses strictly after the given coordinate", - ); + #[test] + fn cycle_frame_is_storage_only() { + // The frame carries ONLY storage identity — no rung / projection / branch + // / semantic tags are minted in this layer (they live upstream). + let f = CycleFrame::new(CycleId(1), DatasetVersion(7)); + assert_eq!(f.cycle, CycleId(1)); + assert_eq!(f.base_version, DatasetVersion(7)); + } + + #[test] + fn order_cycle_stably_is_stable_and_generic() { + // Scrambled → ordered by the supplied key; equal keys keep arrival order. + let mut rows = vec![(3u64, "d"), (1u64, "b1"), (2u64, "c"), (1u64, "b2")]; + order_cycle_stably(&mut rows, |r| r.0); + assert_eq!(rows, vec![(1, "b1"), (1, "b2"), (2, "c"), (3, "d")]); } } From 7e7928a86e2b8573898b47dedff5ef566990f54b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 01:44:18 +0000 Subject: [PATCH 8/9] fix(planner): address CodeRabbit review on persist_sink cycle/WAL reshape (PR #878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five valid findings + nitpicks, verified against the code and fixed: - recover_and_apply now returns Result (Major): it mutates the owner as it walks the chain, so a mid-chain failure had already advanced the owner's phase for the applied prefix but dropped the accumulated Recovered on Err — the caller could not persist the watermark it earned, replaying applied moves against an advanced owner (permanent StalePhase stall acyclic / double-apply cyclic). Each error path now returns the partial Recovered. New falsifier a_mid_chain_failure_returns_the_applied_ prefix_watermark. - PersistError::CycleMismatch (Minor): a cast whose cycle != frame.cycle is a permanent caller error, not the RETRYABLE Write class (retry would loop). New variant + a_cast_for_the_wrong_cycle_is_a_permanent_cycle_mismatch_not_write. - stream_position cross-cycle monotonicity (Major, doc gap): documented on SweepSlot::stream_position + Recovered::watermark that the key is monotonic per owner ACROSS cycles (recover_and_apply compares it over the multi-cycle scan_sealed stream). New falsifier recovery_watermark_spans_multiple_sealed_ cycles. - PersistError::source() (nitpick): wrapping variants (Write/Illegal) now forward to their inner error so callers can walk the chain. - Sealed-predecessor test was tautological (nitpick): FakeWalSink::commit_cycle now fences on a stale base (optimistic-concurrency: base must equal the sealed head), and the test asserts a stale-base commit is REJECTED instead of restating the constructor input — the epistemic horizon is now proven, not assumed. - No-step watermark branch (nitpick): new falsifier a_no_step_landing_advances_ the_watermark_and_is_not_re_scanned (mixed step/no-step chain). - WalSink futures documented as non-Send (nitpick). - LATEST_STATE wording corrected: only the write-side deinterlace_write additions were reverted; temporal.rs's existing query-time causal helpers (LocalCausalRow/local_trajectories) landed in earlier PR commits and remain — the reshape commit leaves temporal.rs untouched, NOT the whole PR. Not actioned (false positive): the "EPIPHANIES append-only" flag on lines 35-37 — those are the unchanged 2026-08-01 entry's Corrections; the new entry is prepended at the top (lines 1-17). The diff mis-attributes because prepending shifts line numbers. lance-graph-planner: 348 lib tests green (+4), clippy + fmt clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 2 +- .../lance-graph-planner/src/persist_sink.rs | 345 ++++++++++++++++-- 2 files changed, 313 insertions(+), 34 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 96f4a179..d8cfcc1a 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,7 +1,7 @@ ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3e persistence-sink reshaped to WAL-amortized cycle (one write per sweep) - `lance_graph_planner::persist_sink` — **reshaped in place** (PR #878) from the per-cast durable-witness model to the **cycle/WAL** model (operator ruling on the seam-shape fork surfaced 2026-08-01). The durable unit is the **cycle/sweep**, NOT the cast: 64k concurrent thoughts stage into an owned `Vec`, `persist_cycle(sink, frame, casts)` folds+freezes them, and `WalSink::commit_cycle(base, DetachedCycleBatch)` does **exactly one atomic append → one `DatasetVersion`** (WAL amortization — 64k thoughts, one write). Types: `CycleId`, `CycleFrame{cycle, base_version}` (storage-only — NO rung/branch/semantic tags), `SweepSlot{cycle, stream_position, owner, row, paired_move, payload}` (boring landing, no `basis`), `LandedSlot{version, slot}`, `DetachedCycleBatch{frame, landings, image}` (frozen, deinterlaced, coalesced). **Write-side ordering** = `order_cycle_stably(rows, key)` (generic over the caller's canonical key; stable-orders casts by the EXISTING `stream_position` in `freeze` BEFORE the append) — completion order never becomes storage order (physical race); `scan_sealed` returns stored order and NEVER sorts. **Epistemic horizon** = sealed read (`read Vn / write Vn+1`); an uncommitted cycle is invisible to `scan_sealed`. **Coalescing** = real per-row fold (`row -> last payload in stream order`), not last-chunk-wins. `recover_and_apply(owner, sealed, applied_through)` + the durable **watermark** idempotence + `StalePhase`/`OwnerMismatch` guards survive unchanged. `versions()` = the cheap coarse timeline a downstream time-series consumer (stockfish-rs, another session) looks up, no landing replay. -- **Write-side ordering lives HERE, not in `temporal.rs`** — that module owns query-time reader-horizon/version reading; the write-side `deinterlace_write` additions made there earlier this session were **reverted** (temporal.rs is untouched — `git diff` clean). The rung a cycle aligns at is decided by whoever schedules it, never modelled in `CycleFrame`. +- **Write-side ordering lives HERE, not in `temporal.rs`** — that module owns query-time reader-horizon/version reading; only the write-side `deinterlace_write` additions made there earlier this session were **reverted**. `temporal.rs`'s existing query-time causal helpers (`LocalCausalRow`, `local_trajectories`/`local_trajectory_of`, layer-1 causal deinterlace) landed in the PR's earlier commits and **remain** — the reshape commit itself leaves `temporal.rs` untouched (this commit's `git diff` on it is empty), NOT the whole PR. The rung a cycle aligns at is decided by whoever schedules it, never modelled in `CycleFrame`. - **Retired:** `DurableWitness`, `DurableReceipt`, `DurableCoordinate`, `DurableWrite`, `persist_cast`, `apply_durable_step`, `scan_witnesses`, `LandedWitness` (the per-cast surface). Vocabulary reused, not re-minted: `DatasetVersion`, `KanbanMove`/`KanbanColumn`, `MailboxId`/`MailboxSoaOwner`, `stream_position`. **No concrete Lance sink built** (deferred per operator, gated on crash falsifiers). Tests are storage/race CONTRACT probes over an in-process `FakeWalSink`, honestly labelled — real MemWAL/restart/atomic-append durability UNPROVEN. See `EPIPHANIES.md` E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1. ## 2026-08-01 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3d persistence-sink durable-witness reshape + temporal layer-1 diff --git a/crates/lance-graph-planner/src/persist_sink.rs b/crates/lance-graph-planner/src/persist_sink.rs index 46976385..4b7805c5 100644 --- a/crates/lance-graph-planner/src/persist_sink.rs +++ b/crates/lance-graph-planner/src/persist_sink.rs @@ -129,6 +129,14 @@ pub struct SweepSlot { pub cycle: CycleId, /// The existing canonical (textual/stream) order key — the write-side /// deinterlace input. NOT a new coordinate. + /// + /// **Contract: monotonically increasing per owner ACROSS cycles, not + /// per cycle.** [`recover_and_apply`] uses it as the durable watermark over a + /// multi-cycle [`WalSink::scan_sealed`] stream (`stream_position <= watermark` + /// ⇒ already applied), so a per-cycle restart to 0 would make recovery skip + /// every later-cycle landing at or below an earlier cycle's watermark. The + /// caller owns this monotonicity (it is the witness-fabric order key, already + /// monotonic); this layer does not re-key by `(CycleId, stream_position)`. pub stream_position: u64, /// The mailbox this landing is on behalf of. pub owner: MailboxId, @@ -202,8 +210,16 @@ impl DetachedCycleBatch { /// Why a persist / seal / step operation produced no lifecycle advance. #[derive(Debug, Clone, PartialEq, Eq)] pub enum PersistError { - /// The cycle seal did not land — nothing published, so no step. + /// The cycle seal did not land — nothing published, so no step. This is the + /// RETRYABLE class (a fenced / failed append may succeed on retry). Write(WriteFailed), + /// A cast was staged against a different cycle than the frame — a caller + /// programming error, PERMANENT and never retryable (distinct from [`Write`] + /// so retry logic never loops on it). [`Write`]: PersistError::Write + CycleMismatch { + cast_cycle: CycleId, + frame_cycle: CycleId, + }, /// The paired move is not a legal Rubicon edge from the owner's current phase. Illegal(RubiconTransitionError), /// A move minted for a different mailbox than the landing's owner — refused so @@ -226,6 +242,13 @@ impl std::fmt::Display for PersistError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Write(e) => write!(f, "{e}"), + Self::CycleMismatch { + cast_cycle, + frame_cycle, + } => write!( + f, + "cast cycle {cast_cycle:?} != frame cycle {frame_cycle:?}" + ), Self::Illegal(e) => { write!(f, "illegal Rubicon transition {:?} -> {:?}", e.from, e.to) } @@ -246,7 +269,21 @@ impl std::fmt::Display for PersistError { } } } -impl std::error::Error for PersistError {} +impl std::error::Error for PersistError { + /// Expose the wrapped cause so callers can walk the error chain — the two + /// wrapping variants ([`Write`](PersistError::Write) / + /// [`Illegal`](PersistError::Illegal)) forward to their inner error; the + /// self-describing variants have no deeper cause. + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Write(e) => Some(e), + Self::Illegal(e) => Some(e), + Self::CycleMismatch { .. } | Self::OwnerMismatch { .. } | Self::StalePhase { .. } => { + None + } + } + } +} /// The WAL seam — **one durable append per cycle**. [`commit_cycle`](WalSink::commit_cycle) /// takes an ALREADY-deinterlaced, ALREADY-frozen [`DetachedCycleBatch`] (the loom @@ -257,6 +294,11 @@ impl std::error::Error for PersistError {} /// /// `&self` (shared) and **no owner borrow**: the persistence path never holds the /// SoA owner across I/O. +/// +/// **Futures are not `Send`** (native `async fn` in trait, no `Send` bound): callers +/// drive these on the crate's single-task persistence path and must NOT +/// `tokio::spawn` them onto a multi-threaded runtime. If a concrete sink ever needs +/// to be spawned, give it explicit `impl Future + Send` methods there. #[allow(async_fn_in_trait)] pub trait WalSink { /// The SINGLE amortized WAL append for a whole cycle: commit `batch` (already @@ -297,10 +339,11 @@ pub async fn persist_cycle( ) -> Result { for c in &casts { if c.cycle != frame.cycle { - return Err(PersistError::Write(WriteFailed(format!( - "cast cycle {:?} != frame cycle {:?}", - c.cycle, frame.cycle - )))); + // Permanent caller error — NOT a retryable Write (retry would loop). + return Err(PersistError::CycleMismatch { + cast_cycle: c.cycle, + frame_cycle: frame.cycle, + }); } if let Some(mv) = c.paired_move { if mv.mailbox != c.owner { @@ -325,7 +368,10 @@ pub async fn persist_cycle( pub struct Recovered { pub applied: Vec, /// The highest `stream_position` now accounted for (per owner), or the input - /// watermark when nothing was applied. + /// watermark when nothing was applied. Same cross-cycle scope as + /// [`SweepSlot::stream_position`] — monotonic per owner across sealed cycles, + /// so it is a durable watermark over the whole multi-cycle `scan_sealed` stream, + /// not a per-cycle counter. pub watermark: Option, } @@ -339,12 +385,19 @@ pub struct Recovered { /// sort is done here. The watermark — not phase equality — is the idempotence key /// (the Rubicon lifecycle is cyclic). Above the watermark the chain must be /// contiguous: a non-matching `from` is a gap/corruption ([`PersistError::StalePhase`]). -/// On error the owner is left mid-chain; re-drive from the persisted watermark. +/// +/// **On error the accumulated [`Recovered`] is returned WITH the error** +/// (`Err((partial, cause))`). The function mutates `owner` as it walks the chain, +/// so a mid-chain failure has already advanced the owner's phase for the applied +/// prefix; returning the partial lets the caller persist the watermark it earned. +/// Discarding it would leave the watermark below the applied prefix, replaying those +/// moves against an already-advanced owner (a permanent `StalePhase` stall in the +/// acyclic case, a double-apply in a cyclic lap). pub fn recover_and_apply( owner: &mut O, sealed: &[LandedSlot], applied_through: Option, -) -> Result { +) -> Result { let mut applied = Vec::new(); let mut watermark = applied_through; let me = owner.mailbox_id(); @@ -356,23 +409,33 @@ pub fn recover_and_apply( match ls.slot.paired_move { None => watermark = Some(ls.slot.stream_position), Some(mv) => { - if mv.mailbox != owner.mailbox_id() { - return Err(PersistError::OwnerMismatch { - move_owner: mv.mailbox, - landing_owner: owner.mailbox_id(), - }); + if mv.mailbox != me { + return Err(( + Recovered { applied, watermark }, + PersistError::OwnerMismatch { + move_owner: mv.mailbox, + landing_owner: me, + }, + )); } if mv.from != owner.phase() { - return Err(PersistError::StalePhase { - owner_phase: owner.phase(), - move_from: mv.from, - }); + return Err(( + Recovered { applied, watermark }, + PersistError::StalePhase { + owner_phase: owner.phase(), + move_from: mv.from, + }, + )); + } + match owner.try_advance_phase(mv.to) { + Ok(step) => { + applied.push(step); + watermark = Some(ls.slot.stream_position); + } + Err(e) => { + return Err((Recovered { applied, watermark }, PersistError::Illegal(e))) + } } - let step = owner - .try_advance_phase(mv.to) - .map_err(PersistError::Illegal)?; - applied.push(step); - watermark = Some(ls.slot.stream_position); } } } @@ -506,17 +569,27 @@ mod tests { impl WalSink for FakeWalSink { async fn commit_cycle( &self, - _base: DatasetVersion, + base: DatasetVersion, batch: DetachedCycleBatch, ) -> Result { if !self.succeed { return Err(WriteFailed("cycle fenced".into())); } + let mut sealed = self.sealed.lock().unwrap(); + // Optimistic-concurrency fence: a commit MUST target the current sealed + // head (`Vn`). A stale/in-flight `base` (a sibling that read an older + // predecessor) is rejected — the epistemic horizon enforced, not assumed. + let head = sealed.last().map_or(DatasetVersion(0), |s| s.version); + if base != head { + return Err(WriteFailed(format!( + "stale base {base:?}: sealed head is {head:?}" + ))); + } // The batch arrives ALREADY deinterlaced + coalesced (loom before WAL). // THE single amortized WAL append for the whole cycle. self.wal_writes.fetch_add(1, Ordering::SeqCst); let version = DatasetVersion(self.next_version.fetch_add(1, Ordering::SeqCst)); - self.sealed.lock().unwrap().push(SealedCycle { + sealed.push(SealedCycle { frame: batch.frame, version, landings: batch.landings, @@ -737,7 +810,7 @@ mod tests { #[tokio::test] async fn a_cycle_reads_the_sealed_predecessor_not_an_in_flight_sibling() { let sink = FakeWalSink::new(); - // Cycle 1 seals → V1. + // Cycle 1 seals → V1 (the sealed head advances to V1). let s1 = persist_cycle( &sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), @@ -745,15 +818,33 @@ mod tests { ) .await .unwrap(); - // Cycle 2's base_version is the SEALED V1 — never an in-flight value. - let f2 = CycleFrame::new(CycleId(2), s1); + assert_eq!(s1, DatasetVersion(1)); + // A sibling that read the STALE predecessor V0 (an in-flight base, not the + // sealed head V1) is FENCED — the sink rejects the commit, proving the + // horizon is enforced, not merely restated by the caller. + let stale = persist_cycle( + &sink, + CycleFrame::new(CycleId(2), DatasetVersion(0)), + vec![slot(99, 2, 0, 0, None)], + ) + .await; + assert!( + matches!(stale, Err(PersistError::Write(_))), + "committing against a stale/in-flight base is fenced, not silently accepted", + ); assert_eq!( - f2.base_version, s1, - "the next cycle reads the SEALED predecessor version (Vn), not in-flight state", + sink.version_count(), + 1, + "the fenced commit published nothing" ); - let s2 = persist_cycle(&sink, f2, vec![slot(99, 2, 0, 0, None)]) - .await - .unwrap(); + // Committing against the SEALED head V1 succeeds and publishes exactly V2. + let s2 = persist_cycle( + &sink, + CycleFrame::new(CycleId(2), s1), + vec![slot(99, 2, 0, 0, None)], + ) + .await + .unwrap(); assert_eq!(s2, DatasetVersion(2), "and publishes exactly its own Vn+1"); } @@ -890,6 +981,194 @@ mod tests { ); } + // ── FALSIFIER: a no-step landing advances the watermark past itself ────────── + #[tokio::test] + async fn a_no_step_landing_advances_the_watermark_and_is_not_re_scanned() { + // A mixed chain: step@0, a NO-STEP landing@1, step@2. If the no-step + // landing did not advance the watermark, it would be re-scanned forever and + // block the watermark behind every following step. + let sink = FakeWalSink::new(); + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![ + slot( + 42, + 1, + 0, + 0, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + ), + slot(42, 1, 1, 1, None), // no-step landing at position 1 + slot( + 42, + 1, + 2, + 2, + Some((KanbanColumn::CognitiveWork, KanbanColumn::Evaluation)), + ), + ], + ) + .await + .unwrap(); + let sealed = sink.scan_sealed(None).await.unwrap(); + let mut o = owner(KanbanColumn::Planning); + let rec = recover_and_apply(&mut o, &sealed, None).unwrap(); + assert_eq!(rec.applied.len(), 2, "the two real steps applied"); + assert_eq!( + rec.watermark, + Some(2), + "the watermark reaches the last landing, past the no-step at 1", + ); + // A second pass with that watermark applies nothing (no re-scan of the no-step). + let again = recover_and_apply(&mut o, &sealed, rec.watermark).unwrap(); + assert!(again.applied.is_empty(), "idempotent — nothing re-applied"); + } + + // ── FALSIFIER: a mid-chain failure returns the watermark it already earned ─── + #[tokio::test] + async fn a_mid_chain_failure_returns_the_applied_prefix_watermark() { + // step@0 is valid and advances the owner; step@1 has a `from` that no longer + // matches (corruption) → StalePhase. The Err must carry the partial Recovered + // so the caller can persist the watermark for the prefix that DID apply. + let sink = FakeWalSink::new(); + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![ + slot( + 42, + 1, + 0, + 0, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + ), + // Above-watermark landing whose `from` (Planning) mismatches the + // owner's now-current phase (CognitiveWork) → StalePhase. + slot( + 42, + 1, + 1, + 1, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + ), + ], + ) + .await + .unwrap(); + let sealed = sink.scan_sealed(None).await.unwrap(); + let mut o = owner(KanbanColumn::Planning); + let (partial, err) = + recover_and_apply(&mut o, &sealed, None).expect_err("the mid-chain mismatch must fail"); + assert!(matches!(err, PersistError::StalePhase { .. })); + assert_eq!( + partial.applied.len(), + 1, + "step 0 applied before the failure" + ); + assert_eq!( + partial.watermark, + Some(0), + "the returned watermark covers exactly the applied prefix (position 0)", + ); + assert_eq!( + o.phase(), + KanbanColumn::CognitiveWork, + "the owner's phase advanced for the applied prefix — the reason the \ + partial watermark must be persistable", + ); + } + + // ── FALSIFIER: the watermark spans cycles (stream_position is cross-cycle) ─── + #[tokio::test] + async fn recovery_watermark_spans_multiple_sealed_cycles() { + // stream_position is monotonic per owner ACROSS cycles: cycle 1 carries + // position 0, cycle 2 carries position 1. Recovery over the multi-cycle + // scan_sealed stream must treat the watermark as cross-cycle. + let sink = FakeWalSink::new(); + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![slot( + 42, + 1, + 0, + 0, + Some((KanbanColumn::Planning, KanbanColumn::CognitiveWork)), + )], + ) + .await + .unwrap(); + persist_cycle( + &sink, + CycleFrame::new(CycleId(2), DatasetVersion(1)), + vec![slot( + 42, + 2, + 1, + 1, + Some((KanbanColumn::CognitiveWork, KanbanColumn::Evaluation)), + )], + ) + .await + .unwrap(); + let sealed = sink.scan_sealed(None).await.unwrap(); + assert_eq!(sealed.len(), 2, "landings from BOTH sealed cycles"); + + // A fresh recovery applies the whole cross-cycle chain; watermark = 1. + let mut o = owner(KanbanColumn::Planning); + let rec = recover_and_apply(&mut o, &sealed, None).unwrap(); + assert_eq!( + rec.applied.iter().map(|m| m.to).collect::>(), + vec![KanbanColumn::CognitiveWork, KanbanColumn::Evaluation], + "both cycles' steps applied in cross-cycle stream order", + ); + assert_eq!( + rec.watermark, + Some(1), + "watermark reaches the cycle-2 landing" + ); + + // A watermark from cycle 1 (position 0) skips ONLY cycle 1, still applies + // cycle 2 — proving the watermark is not per-cycle. + let mut o2 = owner(KanbanColumn::CognitiveWork); + let rec2 = recover_and_apply(&mut o2, &sealed, Some(0)).unwrap(); + assert_eq!( + rec2.applied.iter().map(|m| m.to).collect::>(), + vec![KanbanColumn::Evaluation], + "cycle-1 landing skipped by watermark 0; cycle-2 landing still applied", + ); + } + + // ── FALSIFIER: a cast for the wrong cycle is a PERMANENT error, not Write ──── + #[tokio::test] + async fn a_cast_for_the_wrong_cycle_is_a_permanent_cycle_mismatch_not_write() { + // A cast whose cycle != frame.cycle is a caller programming error — it must + // NOT surface as the RETRYABLE Write class (retry would loop forever). + let sink = FakeWalSink::new(); + let r = persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![slot(42, 2, 0, 0, None)], // cast cycle 2 ≠ frame cycle 1 + ) + .await; + assert!( + matches!( + r, + Err(PersistError::CycleMismatch { + cast_cycle: CycleId(2), + frame_cycle: CycleId(1), + }), + ), + "wrong-cycle cast is CycleMismatch (permanent), never Write (retryable)", + ); + assert_eq!( + sink.wal_writes(), + 0, + "nothing written on a wrong-cycle cast" + ); + } + #[tokio::test] async fn a_cross_owner_move_is_rejected_before_the_cycle_persists() { let bad = SweepSlot { From 6d303e436e06804c6db7c064145cfcf77a08de9e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 08:14:21 +0000 Subject: [PATCH 9/9] docs(plan): record #878 role as primitive cycle/WAL bootstrap + two-dimensional temporal upgrade path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documentation-only. No Rust code, tests, public APIs, temporal.rs, or the #878 persistence implementation change. New narrowly-scoped plan persistence-cycle-wal-bootstrap-v1.md (no suitable canonical plan existed — the persist_sink cycle-WAL seam was tracked only on the board/EPIPHANIES). It records the architectural ruling: - #878 is a PRIMITIVE cycle/WAL bootstrap so thinking can be wired and runnable before the full cognitive topology is solved: concurrent results → primitive slot collection/order → freeze one cycle → one amortized WAL write → one DatasetVersion. Six hard guarantees documented (single sealed Vn read horizon; open-cycle results invisible as Vn input; freeze-before-I/O; one seal = one WAL write = one version; no live mutable SoA borrow to the WAL). - The scalar slot/order model is EXPLICITLY provisional — sufficient for the execution/durability plumbing, not claimed as the final temporal representation. - Two orthogonal dimensions: horizontal = temporal.rs coherence within a frame (deterministic book→chapter→verse→span; partially-ordered neighbourhoods for medical/higher-order thought, so a scalar key is insufficient); vertical = DatasetVersion succession + cheap Stockfish-style hindsight lookup. The version table is vertical only, never horizontal causal ordering. - Planned upgrade: a wait-free-emit shadow temporal.rs coherence pass feeding revision.rs forward-correction into a LATER cycle; a sealed historical version is never silently rewritten. - Known limitations (§4) and scope exclusions (§5) recorded: provisional scalar key, unfinalized conflict semantics, concrete-sink hardening still needed, fake sink proves contract shape not crash durability; no cohort internals / slot count / actor-neighbour waiting; no new semantic/temporal/rung/witness/branch/ ancestry types; no reviving ThoughtWitness/basis/awareness_seq/per-cast WAL. Cross-references (does not re-specify): horizontal detail → temporal-markov-and-style-classes-v1.md; D-MBX-A6 tracking → STATUS_BOARD; per-row write gate → mailbox-cycle-aware-write-contract-v1.md; reshape ruling → EPIPHANIES. Registered in INTEGRATION_PLANS.md (prepended, newest-first). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/INTEGRATION_PLANS.md | 26 ++ .../persistence-cycle-wal-bootstrap-v1.md | 224 ++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 .claude/plans/persistence-cycle-wal-bootstrap-v1.md diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 291a4279..0ba26e5a 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -38,6 +38,32 @@ vectors → four Ψ ripple fields; cloned-lens control as the can-it-stay-silent falsifier), LLM tail last. deepnsm-v2 crate untouched by ruling. A withdrawn probe verdict (tesseract-side, retention-axis category error) is recorded in §8's provenance + tesseract-rs CLAUDE.md. +## 2026-08-02 — persistence-cycle-wal-bootstrap v1 — ACTIVE (bootstrap SHIPPED #878; temporal/revision upgrade PLANNED) — main thread + +**Plan:** `.claude/plans/persistence-cycle-wal-bootstrap-v1.md` +Documentation-only architectural ruling recording the *role* of the #878 +persistence seam and the larger two-dimensional temporal architecture it +bootstraps toward. #878 is a **primitive cycle/WAL bootstrap** (concurrent +results → primitive slot collection/order → freeze one cycle → one amortized +WAL write → one `DatasetVersion`), with six hard guarantees (single sealed `Vn` +read horizon; open-cycle results invisible as `Vn` input; freeze-before-I/O; one +seal = one WAL write = one version; no live mutable SoA borrow to the WAL). The +scalar slot/order model is **explicitly provisional** — sufficient for the +execution/durability plumbing, not the final temporal representation. Two +orthogonal dimensions: **horizontal** = `temporal.rs` coherence within a frame +(book→chapter→verse→span for deterministic streams; partially-ordered +neighbourhoods for medical/higher-order thought — hence a scalar key is +insufficient); **vertical** = `DatasetVersion` succession + cheap +Stockfish-style hindsight lookup. Planned upgrade: a wait-free-emit shadow +`temporal.rs` coherence pass feeding `revision.rs` forward-correction into a +later cycle — a sealed version is never silently rewritten. Cross-refs (does not +re-specify): horizontal detail → `temporal-markov-and-style-classes-v1.md`; +D-MBX-A6 tracking → STATUS_BOARD; per-row write gate → +`mailbox-cycle-aware-write-contract-v1.md`; reshape ruling → EPIPHANIES +`E-THE-DURABLE-UNIT-IS-THE-CYCLE-...-1`. Scope exclusions: no cohort internals / +slot count / actor-neighbour waiting; no new semantic/temporal/rung/witness/ +branch/ancestry types; no reviving ThoughtWitness/basis/awareness_seq/per-cast +WAL. ## 2026-07-26 — rosetta-codebook-convergence v1 — PROPOSED (D-RCC-1 calibrator runnable today) — main thread diff --git a/.claude/plans/persistence-cycle-wal-bootstrap-v1.md b/.claude/plans/persistence-cycle-wal-bootstrap-v1.md new file mode 100644 index 00000000..100eb94c --- /dev/null +++ b/.claude/plans/persistence-cycle-wal-bootstrap-v1.md @@ -0,0 +1,224 @@ +# persistence-cycle-wal-bootstrap-v1 — the primitive cycle/WAL seam and its temporal/revision upgrade path + +> **Status:** ACTIVE (bootstrap SHIPPED in PR #878; upgrade phases PLANNED). +> **Date:** 2026-08-02. +> **Scope:** documentation-only architectural ruling. Records the *role* of the +> #878 persistence seam and the intended larger two-dimensional temporal +> architecture it bootstraps toward. Changes **no** Rust code, tests, public +> APIs, or `temporal.rs`. +> **Owns (narrowly):** "what #878 is and is NOT", the horizontal/vertical +> temporal split as it touches persistence, and the later +> `temporal.rs`-shadow + `revision.rs` correction path. +> **Does NOT own (cross-refs, never re-specifies):** +> - Horizontal temporal-stream detail → `temporal-markov-and-style-classes-v1.md` +> (the ratified 2026-07-10 arc; `E-MARKOV-TEMPORAL-STREAM-1`). +> - D-MBX-A6 deliverable tracking → `.claude/board/STATUS_BOARD.md` +> (D-MBX-A6-P1…P3e rows). +> - Per-row `write_row` cycle-gate → `mailbox-cycle-aware-write-contract-v1.md` +> (a *different* deliverable — the SoA setter gate, not the WAL seam). +> - The reshape ruling itself → `.claude/board/EPIPHANIES.md` +> `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1` +> and `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1`. +> +> This is the bootstrap-and-upgrade companion, not a competing architecture. + +--- + +## 1. Current #878 role — a PRIMITIVE cycle/WAL bootstrap (deliberate) + +#878 deliberately establishes a **primitive** persistence seam so that thinking +can become *wired and runnable* without first solving the complete cognitive +topology. It is scaffolding that carries load, chosen so the execution and +durability plumbing exists and is exercised **before** the final temporal or +cognitive representation is settled. + +Its accepted responsibilities, end to end: + +``` +concurrent thought results + → primitive slot collection and ordering + → freeze one complete cycle + → one amortized WAL write + → one Lance DatasetVersion +``` + +The **hard guarantees** #878 makes (and only these): + +1. All work in a cycle reads exactly **one sealed predecessor version `Vn`**. +2. **Open-cycle results are not visible as `Vn` input** (an unsealed cycle is + excluded from the sealed read horizon). +3. The cycle is **frozen before durable I/O** (order + coalesce happen on a + detached snapshot, then the append runs). +4. One successful cycle seal produces **exactly one WAL write**. +5. One successful cycle seal produces **exactly one `DatasetVersion`**. +6. The WAL **never receives a live mutable SoA borrow** (the persistence path + holds no owner across I/O). + +### The scalar slot/order model is explicitly PROVISIONAL + +The current scalar slot + single-key ordering model is **sufficient** to +establish the execution and durability plumbing, and **nothing more**. It is +**not** claimed to be the final representation of temporal or cognitive order. +This document does **not** redesign or fix that limitation — recording the +boundary is the whole point. The upgrade path is §2–§3; the accepted debts are +§4. + +--- + +## 2. The intended larger architecture — two orthogonal dimensions + +The final architecture is two **orthogonal** dimensions. #878 supplies the +vertical axis primitively and leaves the horizontal axis to a later phase. + +### 2.1 Horizontal dimension — coherence within a frame (`temporal.rs`) + +`temporal.rs` preserves and reconstructs **horizontal causal and contextual +coherence** — the relationships *among* the results that make up one frame. + +For **deterministic streamed material** (e.g. the Bible), the initial +horizontal position is strongly determined by the source structure: + +``` +book → chapter → verse → token/span +``` + +Preserving this horizontal structure is required because it is what the local +awareness reads: + +``` +signed i4 neighbourhood pointers + → relative-pronoun and anaphora relations + → local grammatical and causal relationships + → coherent standing-wave awareness +``` + +For **medical knowledge and higher-order thought**, horizontal relationships +may **not** be monotonically ordered — they may form **concurrent or partially +ordered neighbourhoods**. The final temporal architecture must therefore be able +to resolve **more than a scalar total order**. (This is precisely why the #878 +scalar key is labelled provisional in §1.) + +> The horizontal-stream mechanism itself — the version-range read +> (`QueryReference::at(v, rung)` + deinterlace), the ±5-generalizing window, the +> `LocalCausalRow`/`local_trajectories` reconstruction — is owned by +> `temporal-markov-and-style-classes-v1.md` and is **not** re-specified here. +> This document only records *that* the horizontal dimension is `temporal.rs`'s +> responsibility and *why* a scalar key is insufficient for it. + +### 2.2 Vertical dimension — successive durable frames (`DatasetVersion`) + +Lance `DatasetVersion` is the **vertical** axis. Each successful cycle seal +advances the durable frame: + +``` +Vn → Vn+1 → Vn+2 +``` + +The version table therefore provides **both**: + +- **vertical cognitive/rung progression** (each sealed frame is one discrete + advance), and +- **cheap historical time-series lookup** (a version is selected without + replaying any landing). + +Consumers such as **Stockfish-style hindsight tests** can select versions from +the Lance version table because the frames have already been sealed as discrete +durable states — the lookup is over already-coherent frames. + +> The version table performs **vertical** frame succession and lookup **only**. +> It does **not** perform horizontal causal ordering — that is the horizontal +> dimension's job (§2.1). Conflating the two is the error this section forecloses. + +--- + +## 3. Planned temporal error-correction phase (later; not #878) + +The primitive #878 slot model may initially produce frames whose **horizontal +coherence is incomplete** — the scalar order captures *that* results happened, +not the full partial-order structure among them. + +A later phase may run `temporal.rs` as a **metacognitive / shadow correction +layer** over the produced frame. This layer **must not** require ractor actors +to wait synchronously for neighbours during thought execution — the emit path +stays wait-free. + +Intended direction: + +``` +actors emit without neighbour waits + → primitive cycle is collected + → temporal coherence is evaluated (shadow pass) + → inconsistencies become correction / revision input + → a LATER cycle carries the corrected interpretation +``` + +`revision.rs` is the **planned complementary mechanism** for reinterpretation, +circular reasoning, and correction from the reflective side. (Distinct from the +NARS *belief-revision* of policy atoms in `triangle-tenants-gestalt-separation-v1.md` +— that revises `LearnedStyle`; this `revision.rs` feeds justified *reinterpretation* +into subsequent cognition.) + +Conceptually, the three responsibilities stay separate: + +``` +temporal.rs detects or reconstructs horizontal coherence +revision.rs feeds justified correction into subsequent cognition +Lance versions preserve the successive durable vertical frames +``` + +**Iron constraint on the correction path:** a sealed historical version **must +not be silently rewritten** merely because a later metacognitive pass revised +its interpretation. Correction flows *forward* — into a later cycle / later +version — never *backward* over a sealed frame. (This is the vertical-axis +immutability that keeps the hindsight-lookup in §2.2 honest.) + +--- + +## 4. Known limitations accepted for #878 + +Recorded explicitly as **upgrade points**, not as claims that #878 already +solves the final temporal model: + +- The scalar slot model is **provisional**. +- A scalar key **may not capture** future partial-order cognition (§2.1). +- Cross-owner and equal-position **conflict semantics are not finalized**. +- Cycle **retry** and **production WAL idempotence** still require + concrete-sink hardening. +- The **fake WAL sink proves the contract shape, not real crash durability** + (`compile+test green ≠ storage proven`, the Ladybug lesson). +- The **complete horizontal temporal projection remains future work** (§2.1, + cross-ref `temporal-markov-and-style-classes-v1.md`). + +--- + +## 5. Scope exclusions (this document and the #878 bootstrap) + +- Do **not** introduce or document detailed **cohort internals**. +- Do **not** mention a **fixed number of cohort slots**. +- Do **not** design **actor-neighbour waiting or firing dependencies** (the + emit path is wait-free by §3). +- Do **not** invent new **semantic, temporal, rung, witness, branch, or + ancestry** types. +- Do **not** revive `ThoughtWitness`, `basis`, `awareness_seq`, or **per-cast + WAL persistence** (all retired by the reshape ruling; the durable unit is the + cycle, not the cast). +- The **cohort execution model** is out of scope — it belongs to the separate + cohort architecture work, not here. + +--- + +## 6. Status snapshot + +| Aspect | State | +|---|---| +| Cycle/WAL bootstrap seam (`persist_sink`) | **SHIPPED** in PR #878 (bootstrap; contract-probed, not storage-proven) | +| Vertical axis (`DatasetVersion` succession + lookup) | Established primitively by the seam | +| Horizontal axis (`temporal.rs` coherence over a frame) | **PLANNED** — owned by `temporal-markov-and-style-classes-v1.md` | +| Shadow temporal-coherence correction pass | **PLANNED** (§3) | +| `revision.rs` forward-correction mechanism | **PLANNED** (§3) | +| Concrete Lance sink (real crash durability) | **DEFERRED** — gated on crash falsifiers (§4) | + +The bootstrap exists so the rest can be built on a running, durable seam. The +scalar order is a load-bearing placeholder, and this document is the record that +it is a placeholder — nothing here claims the primitive ordering is the final +temporal architecture.