From ad9c3ad1256c146fa09589b01b9071a3d30576be Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 08:46:09 +0000 Subject: [PATCH 1/6] docs(plan): sparse-delta cycle ruling + loop-closure driver plan (post-#878 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documentation-only. No Rust code, tests, public APIs, persist_sink.rs, temporal.rs, or the persistence implementation change. 1) Sparse-delta ruling (persistence-cycle-wal-bootstrap-v1.md §2, NEW): "complete logical cycle ≠ full physical dataset rewrite". A globally-complete cycle persists ONLY its coalesced dirty-row delta + required durable transition metadata; unchanged rows are inherited from the sealed base version, never re-serialized because they participated. Records the verbatim storage invariant, the participation-vs-mutation split (a no-mutation participant needs no 512-byte row), the honest in-memory payload-duplication limitation (concrete sink must not persist both per-landing bytes AND the coalesced image), the capacity/backpressure ruling (dense cycle = explicit capacity event), and 5 concrete-sink falsifiers (sparse / no-op-policy / coalescing / dense-capacity / retention). Status: RATIFIED architecture, UNIMPLEMENTED in a concrete Lance sink. Preserves the horizontal(temporal.rs) / vertical(DatasetVersion) / revision.rs split — density only. Sections renumbered (§2 inserted; §3–§7 shifted, all internal §-refs updated). 2) Loop-closure driver plan (cycle-loop-closure-driver-v1.md, NEW): the seam that makes the merged persist_sink load-bearing at 64k — persist_sink has zero production callers today, so the loop is open. Closes collect → persist_cycle → sealed version → sync inline fan-step (on_version + try_advance_phase, NOT 64k async drive_once) → CognitiveWork → owner_adapter → next cycle. Mints no new types. Deliverables D-MBX-A6-P4a..f, probe-first. Home: lance-graph-supervisor. Board: EPIPHANIES E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1 (prepend); INTEGRATION_PLANS cycle-loop-closure-driver v1 (prepend); STATUS_BOARD D-MBX-A6-P4 row + P3d flipped to Merged (#878, reshaped to cycle/WAL + sparse ruling). Branch restarted from main after #878 merged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 18 + .claude/board/INTEGRATION_PLANS.md | 26 ++ .claude/board/STATUS_BOARD.md | 3 +- .claude/plans/cycle-loop-closure-driver-v1.md | 322 ++++++++++++++++++ .../persistence-cycle-wal-bootstrap-v1.md | 176 +++++++++- 5 files changed, 528 insertions(+), 17 deletions(-) create mode 100644 .claude/plans/cycle-loop-closure-driver-v1.md diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 000bc37e..92206823 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,21 @@ +## 2026-08-02 — E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1 — a globally-complete cycle persists only its coalesced dirty-row delta, never a full 64k-row snapshot + +**Status:** RATIFIED as architecture; UNIMPLEMENTED in a concrete Lance sink (operator ruling, 2026-08-02). **Confidence:** High for the ruling (a storage-density invariant); the concrete sink that must honor it is deferred and unbuilt. Documentation-only — no Rust/tests changed. Recorded in `.claude/plans/persistence-cycle-wal-bootstrap-v1.md` §2. + +**The ruling.** "One complete cycle image" must NEVER be read as serializing every row because every participant belonged to the cycle. **Complete logical cycle ≠ full physical dataset rewrite.** A cycle is *logically complete* when all required participants reached the boundary, all updates were collected + temporally ordered/coalesced, all required lifecycle transitions were included, the change set was frozen, and it committed atomically. The *physical payload stays sparse*: 64k participants → N dirty rows (N may be ≪ 64k) → one frozen sparse delta batch → one WAL transaction → one `DatasetVersion`. Unchanged rows are **inherited from the sealed predecessor version** and MUST NOT be serialized merely because they participated. + +**The invariant (verbatim).** «A cycle is globally complete but physically sparse. `commit_cycle` persists only the coalesced dirty-row set and the required durable transition metadata. Unchanged rows remain inherited from the sealed base version and do not become new row payloads merely because they were members of the cycle.» "One WAL write per cycle" = one atomic durability boundary for the sparse change set, NOT one full 64k-row (~32 MiB) snapshot. + +**Participation ≠ mutation.** Cycle completion evidence (cycle identity, sealed base version, participation digest, dirty-row count, transition count, batch digest) is a small footer, compact + separate from row payloads. A participant that produced no state mutation MUST NOT require a 512-byte row payload merely to prove participation. (Cohort internals — participant-count encoding, bitmap layout, ownership — are out of scope; separate cohort architecture session.) + +**Honest bootstrap limitation.** The #878 contract-probe duplicates bytes in memory (`SweepSlot` owns payload + `DetachedCycleBatch` retains landings + `freeze` clones the final image). Acceptable for the fake; the concrete sink must NOT persist both the per-landing bytes AND the coalesced image. Future concrete shape: landing metadata / durable transitions + ONE detached coalesced dirty-row image + a small cycle footer. (Structs NOT redesigned here — a concrete-sink upgrade requirement.) + +**Capacity is an explicit event.** Normal cycle = sparse delta; worst case (every row genuinely dirty) ≈ one full row slab — VALID but an explicit capacity event, not the default shape. The concrete sink must define (numbers not chosen here): max frozen cycles in flight, max bytes in flight, WAL/storage backpressure, checkpoint/compaction, version-retention, disk-monitoring + refusal threshold. + +**Five concrete-sink falsifiers (deferred, probe-first).** (1) sparse-cycle: 64k participants / 17 dirty → exactly 17 coalesced payloads + one version, rest inherited from Vn; (2) no-op-cycle: zero dirty + zero transitions → ONE documented policy (no new version OR metadata-only version; never a full empty slab); (3) coalescing: many updates one row → one final payload written, no duplicate intermediate row state, transition history separate; (4) dense-cycle capacity: all dirty → one bounded batch, backpressure bounds the queue, no silent disk exhaustion; (5) retention: many versions → documented cleanup bounds disk growth, hindsight-horizon versions stay readable. + +**Preserves the split (unchanged):** horizontal coherence = `temporal.rs`; vertical durable succession = `DatasetVersion`; correction = `revision.rs` into a later version; execution/cohort internals = separate work. This ruling concerns physical persistence density ONLY — it does not pull cohort topology, temporal partial ordering, or revision semantics into the concrete-sink design. Extends `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`. + ## 2026-08-02 — E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1 — the persistence seam is reshaped to the WAL-amortized generation; per-cast durability retired **Status:** FINDING (operator-ruled 2026-08-02 — the decision on the fork Correction 2(c) of `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` surfaced). **Confidence:** High for the **storage/race CONTRACT** (`lance-graph-planner` 344 lib tests, 13 in `persist_sink`, clippy+fmt clean); crash-durability remains contract-probed over an in-process fake, NOT storage-proven (`compile+test green ≠ storage proven`, the Ladybug lesson). Builds NO concrete sink. diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 0ba26e5a..de716fcf 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,29 @@ +## 2026-08-02 — cycle-loop-closure-driver v1 — PLANNED / CONJECTURE (the seam that makes persist_sink load-bearing at 64k) — main thread + +**Plan:** `.claude/plans/cycle-loop-closure-driver-v1.md` +The loop-closure driver: the missing seam that turns the merged `persist_sink` +cycle/WAL bootstrap into a running loop at 64k concurrency. Today +`persist_sink::{persist_cycle, WalSink, versions}` has **zero production +callers** — the loop is open. The driver closes `collect casts → persist_cycle → +sealed DatasetVersion → sync fan-step across the mailbox fleet → +try_advance_phase (the KanbanStep) → CognitiveWork runs the thought → +owner_adapter casts the next intent → back to collect`. Correctness pivot: the +driver WROTE the version, so it fires `NextPhaseScheduler::on_version` + +`try_advance_phase` **inline and synchronously** (no dataset re-read) — NOT 64k +async `LanceVersionScheduler::drive_once` (that subscription variant is for +reading a version you did NOT write). Mints NO new types — composes +`KanbanMove`/`DatasetVersion`/`SweepSlot`/`BatchWriter`/`NextPhaseScheduler`/ +`KanbanActor`/`owner_adapter`/`recover_and_apply`. Deliverables D-MBX-A6-P4a +(driver skeleton) → P4b (fleet fan-step) → P4c (loop closure round-trip) → P4d +(wait-free-emit guard) → P4e (recovery composition) → P4f (16k/64k scale, +W2a-gated), each probe-first. Home: `lance-graph-supervisor` (structural fleet +owner; new planner path-dep, no cycle) with a planner fallback. HONEST: the +CONTROL loop closes; the durability leg stays the contract-probe fake until the +concrete `LanceShardSink` lands. Board-as-tenant (D-V3-W2a) is a SCALE gate, not +a control-loop blocker. Companion to `persistence-cycle-wal-bootstrap-v1.md` +(which also gained the §2 sparse-delta storage ruling this session — see +EPIPHANIES `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). + ## 2026-08-01 — CORRECTION to the §8 entry below: `RungLevel 0–10` → `0–9` The `2026-07-31 — SYNERGY-MAP-S00-S07 §8` entry's summary line reads diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 0270d578..f873555e 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -963,7 +963,8 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3a | StyleStrategy: thinking-style -> cluster -> mechanism -> recipe_kernels Tactic selection (planning substrate; carries tau JIT addr) | lance-graph-planner | 130 | LOW | **In PR** | #439; first cut of A6-P3 consumer wiring; planner now consumes contract recipes/styles; deferred: i4-32D decode, Outcome->Candidate, tau->JIT, membrane commit | | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | -| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **In PR** | #878; ordering/recovery CONTRACT probed (349 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | +| D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **Merged** | #878 (merged; reshaped in place to the cycle/WAL model = P3e — `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`; + the §2 sparse-delta storage ruling `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`, RATIFIED/UNIMPLEMENTED); ordering/recovery CONTRACT probed (348 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | +| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `collect casts → persist_cycle → sealed DatasetVersion → sync fan-step (`NextPhaseScheduler::on_version` + `try_advance_phase`, inline — NOT 64k async `drive_once`) → CognitiveWork → `owner_adapter` casts next intent → loop`. Mints NO new types. Sub-deliverables P4a (driver skeleton) / P4b (fleet fan-step) / P4c (round-trip closure) / P4d (wait-free-emit) / P4e (recovery) / P4f (16k/64k scale, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **Queued (PLANNED)** | plan `.claude/plans/cycle-loop-closure-driver-v1.md`; home lance-graph-supervisor (new planner dep, no cycle; planner fallback); CONTROL loop closes, durability leg stays the contract-probe fake until concrete `LanceShardSink`; consumes #878 persist_sink; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/.claude/plans/cycle-loop-closure-driver-v1.md b/.claude/plans/cycle-loop-closure-driver-v1.md new file mode 100644 index 00000000..ae5b4c57 --- /dev/null +++ b/.claude/plans/cycle-loop-closure-driver-v1.md @@ -0,0 +1,322 @@ +# cycle-loop-closure-driver-v1 — the loop-closure driver that makes the persist_sink cycle/WAL seam load-bearing + +> **Status:** PLANNED / CONJECTURE — design only. The **CONTROL loop** this +> driver closes is the deliverable; the **durability leg stays the +> contract-probe fake** until the concrete `LanceShardSink` lands (the +> `compile+test green ≠ storage proven` Ladybug rule). Each claim below is +> probe-gated; nothing here is shipped. +> **Date:** 2026-08-02. +> **Scope:** documentation-only architectural ruling. Records the *missing +> seam* — the driver that turns the already-merged `persist_sink` cycle/WAL +> bootstrap into a running loop at 64k concurrency — and the deliverables + +> falsifiers that gate its construction. Changes **no** Rust code, tests, +> public APIs, `persist_sink.rs`, or `temporal.rs`. +> **Owns (narrowly):** "why the loop is open today", the closed-loop +> seal→step→think→cast shape, the writer-fires-inline correctness point, the +> D-MBX-A6-P4a…f deliverables, the home + dep-direction decision, and the 64k +> mechanics as they bear on the control loop. +> **Does NOT own (cross-refs, never re-specifies):** +> - The cycle/WAL seam itself (the OUT/durability half + two-dimensional +> temporal model) → `persistence-cycle-wal-bootstrap-v1.md`. +> - Horizontal temporal-stream detail (the version-range read, ±5 window) → +> `temporal-markov-and-style-classes-v1.md`. +> - Per-row `write_row` cycle-gate + the 16k-per-prefix scale framing → +> `mailbox-cycle-aware-write-contract-v1.md`. +> - Deliverable tracking → `.claude/board/STATUS_BOARD.md` +> (D-MBX-A6-P1…P3e shipped, D-MBX-9-IN scheduler contract, D-V3-W2a board +> tenant gated, D-V3-W2b supervisor kanban_actor shipped, D2 symbiont +> kanban_loop slice shipped). +> - The reshape rulings → `.claude/board/EPIPHANIES.md` +> `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`, +> `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1`, +> `E-SUBSTRATE-IS-THE-SCHEDULER`. +> +> This is the loop-closure companion to the persistence bootstrap plan, not a +> competing architecture. The driver mints **no** new semantic / temporal / +> rung / witness / branch / ancestry types — it composes existing organs. + +--- + +## 1. The gap — `persist_sink` has ZERO production callers today (the loop is OPEN) + +The organs all exist and are merged. The loop is **not** closed. + +`lance_graph_planner::persist_sink` — `persist_cycle`, the `WalSink` trait +(`commit_cycle` / `scan_sealed` / `versions`), `recover_and_apply` — has, as of +this plan, **zero production callers** (verified by grep). It is a load-bearing +seam with nothing standing on it. The cycle can be frozen, one WAL write can be +made, one sealed `DatasetVersion` can be returned — but *nothing calls it in a +running loop*, so the version it seals never fans out to advance any mailbox, +and no finished thought ever casts the next cycle's intent. + +Everything the loop needs is already built, in five separate crates: + +- **persist** — `persist_sink::persist_cycle(sink, frame, casts)` freezes a + `CycleFrame { cycle, base_version }` + `Vec` into one WAL write and + returns a sealed `DatasetVersion`. Recovery: `recover_and_apply(owner, sealed, + applied_through)`. +- **schedule (sync)** — `lance_graph_contract::scheduler::VersionScheduler::on_version` + + `NextPhaseScheduler` (forward-arc Planning→CognitiveWork→Evaluation→Commit, + Libet −550 µs stamp on the Planning→CognitiveWork Σ-crossing, `None` on + absorbing). +- **schedule (async subscription)** — + `lance_graph::graph::scheduler::LanceVersionScheduler::drive_once` / + `drive_at_latest` — for READING a version you did **not** write (opens the + Lance dataset per call). +- **apply** — `lance_graph_supervisor::kanban_actor::KanbanActor` (the + ractor actor whose State IS the owner; applies via `try_advance_phase`) + + the free fns `drive_version_tick` / `drive_scheduled_tick`. +- **emit** — `lance_graph_planner::owner_adapter::{rebind_bootstrap, + emit_bootstrap_intent}` turns a finished thought's Outcome into the next + cycle's intent cast via `batch_writer::BatchWriter::cast(on_behalf, moves, + payload)`. + +The **shipped slice that proves the shape** is +`symbiont::kanban_loop::SymbiontBoard` (D2): it impls `MailboxSoaView` + +`MailboxSoaOwner` over a `Vec` and its `step(&NextPhaseScheduler)` +drives `version_tick → on_version → try_advance_phase` synchronously, with a +`u32` tick standing in for the real Lance version. The driver **generalizes +that slice** to (a) the real sealed `DatasetVersion` from `persist_cycle` and +(b) a mailbox fleet instead of one `SymbiontBoard`. + +The gap, stated once: **no crate composes persist → schedule → apply → emit into +a running cycle.** That composition is the driver. It is a control-loop, not a +new subsystem. + +--- + +## 2. The closed-loop shape — seal → step → think → cast + +The driver closes exactly this loop: + +``` + collect the fleet's staged BatchWriter casts → Vec + │ + ▼ + persist_cycle(sink, CycleFrame{cycle, base_version=Vn}, casts) + │ (one WAL write, freeze-before-I/O) + ▼ + sealed DatasetVersion Vn+1 + │ + ▼ + fan the step across the mailbox fleet ← writer fires INLINE (§3) + NextPhaseScheduler::on_version(view_i, Vn+1, exec) (sync, pure) + │ + ▼ + try_advance_phase per mailbox ← the KanbanStep (KanbanActor) + │ + ▼ + CognitiveWork runs the thought ← pluggable callback (§5.4 seam) + │ produces an Outcome + ▼ + owner_adapter: Outcome → emit_bootstrap_intent → BatchWriter::cast(on_behalf=owner) + │ + └──────────────── back to collect (next cycle Vn+2) ───────────────┘ +``` + +The KanbanMove is the parcel-address; the step is the delivery-scan +(`E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1`). The durable +unit is the cycle, not the cast +(`E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`). The +substrate — the sealed version table — IS the scheduler +(`E-SUBSTRATE-IS-THE-SCHEDULER`). + +The driver adds **no** node types, **no** move types, **no** scheduler types. It +is glue: it collects the casts the fleet already staged, calls a function that +already exists, and routes the result back through an adapter that already +exists. + +--- + +## 3. Correctness — the writer fires the step INLINE and SYNCHRONOUSLY (not 64k `drive_once`) + +This is the load-bearing correctness point. + +**The driver WROTE the version** — `persist_cycle` returned `Vn+1`. Because the +driver already holds the version it committed, it fires the fan-step **inline +and synchronously**: + +``` +NextPhaseScheduler::on_version(view_i, Vn+1, exec) // sync pure fn, per mailbox + → try_advance_phase(mailbox_i) // per mailbox +``` + +`on_version` is a **sync pure function** (board D2 line: the writer knows the +version it committed and fires the update inline, no async). The sweep is a +straight-line pass over the mailbox SoA — one sync call per mailbox, then the +per-mailbox `try_advance_phase`. There is **no second dataset read** between the +seal and the step. + +The driver does **NOT** fan 64k async `drive_once` / `drive_at_latest` calls. +`LanceVersionScheduler::drive_once` / `drive_at_latest` are the **subscription** +variant — async precisely because they READ a version they did NOT write, and +**each opens the Lance dataset**. Fanning 64k of those across a fleet would be +64k dataset opens to re-read a version the driver already has in hand: wrong, +and quadratically wrong at scale. + +The rule, stated for the record: + +> **Async is ONLY (a) the `persist_cycle` I/O leg and (b) the subscription +> drive path (a reader that did not write the version). The writer-side +> fan-out is sync `on_version` + `try_advance_phase`, inline, no dataset +> re-read.** + +The subscription path (`drive_at_latest`) remains the correct tool for a +*separate* reader process that observes sealed versions it did not produce — +that reader legitimately opens the dataset because it has no other handle to the +version. That is a different actor from the driver and out of scope here. + +--- + +## 4. Deliverables (probe-first — a falsifier per deliverable) + +Per the workspace falsifiability rule: *what input makes this fail?* Every +deliverable names its falsifier; anti-vacuity and can-it-fire / can-it-stay- +silent twins are called out where the naive assertion would be vacuous. + +| ID | Deliverable | Falsifier (what input makes it fail) | +|---|---|---| +| **D-MBX-A6-P4a** | **CycleDriver skeleton** — drain the fleet's `BatchWriter` casts into `Vec` → `persist_cycle` → sealed `DatasetVersion`. | N staged casts produce **exactly one** WAL write + **exactly one** version (reuse `persist_sink`'s amortization probe at the driver level — assert `commit_cycle` invoked once, not N times). | +| **D-MBX-A6-P4b** | **fleet fan-step** — sealed version → sync `on_version` sweep over the mailbox SoA → `try_advance_phase` per mailbox (via `KanbanActor` / owned sweep). | (i) **anti-vacuity:** a fleet with a *mix* of phases — each mailbox advances by exactly one **legal forward-arc** step AND assert different mailboxes took **different** steps (not lockstep-blind); (ii) an **absorbing** mailbox fires nothing (`on_version` → `None`); (iii) assert **NO second dataset read** between seal and step (proves the writer fires inline per §3, not via a re-read `drive_once`). | +| **D-MBX-A6-P4c** | **loop closure** — a CognitiveWork Outcome → `owner_adapter::emit_bootstrap_intent` → `BatchWriter::cast` into the NEXT cycle → appears in Vn+1's collected casts (round-trip). | An Outcome cast in cycle N is **present in cycle N+1's collected casts** AND advances the owner **one step further** (not merely enqueued — actually collected and applied next cycle). | +| **D-MBX-A6-P4d** | **wait-free-emit guard** — a mailbox whose neighbour has NOT completed still advances (no synchronous neighbour wait). | **can-it-fire:** construct a fleet where mailbox B is mid-thought and mailbox A completes — A **still steps in the same cycle**; assert **no barrier** / no neighbour wait blocked A. | +| **D-MBX-A6-P4e** | **recovery composition** — `recover_and_apply` replays the owner's pending tail after a mid-loop stop, idempotent with the watermark. | Stop mid-loop, re-drive, assert **no double-apply** (reuse the `persist_sink` watermark probe at the driver level — `applied_through` gates the replay so a re-applied slot is a no-op). | +| **D-MBX-A6-P4f** *(SCALE, gated on W2a)* | **16k / 64k mailboxes fan in one cycle** within the cycle budget. | **MEASURED**, labelled a **scale gate, not a correctness claim**: 16k/64k mailboxes fan in one cycle within the ~0.5–2.5 s/cycle budget; **log what was measured, never a silent cap**. | + +**Sequencing:** P4a (collect+seal) and P4b (fan-step) are the spine; P4c closes +the round-trip; P4d and P4e are the wait-free + recovery guards on the spine; +P4f is the scale gate, deferred with W2a (§6). + +--- + +## 5. Home, dependency direction, and the WalSink-fake honesty + +### 5.1 HOME — `lance-graph-supervisor` (with a stated fallback) + +**Decision:** the driver lives in **`lance-graph-supervisor`** — the structural +fleet owner. It already owns `KanbanActor` + the owner-apply surface +(`try_advance_phase`, `drive_version_tick`, `drive_scheduled_tick`), which is +exactly the "apply" leg of the loop. Putting the control-loop next to the apply +surface keeps the fan-step where the fleet ownership already is. + +The supervisor crate currently deps **only** `lance-graph-contract` (NOT +planner). The driver requires the planner's `persist_sink`, `owner_adapter`, and +`batch_writer`, so the wiring task adds a **`lance-graph-planner` path-dep** to +supervisor. This is safe: **planner does NOT dep supervisor** (verify with +`cargo tree` before landing), so there is no cycle. + +**Fallback (stated as a decision, not left open):** if adding the planner dep to +supervisor surfaces a cycle (e.g. planner gains a supervisor dep in the +meantime), the driver instead lives **in `lance-graph-planner`** alongside +`persist_sink` / `owner_adapter` / `batch_writer`, and reaches the apply surface +through the contract's `VersionScheduler` + `MailboxSoaOwner` traits rather than +the concrete `KanbanActor`. The control-loop shape (§2, §3) is identical either +way; only the crate boundary moves. + +### 5.2 Dependency direction + +``` +lance-graph-supervisor ──(new path-dep)──► lance-graph-planner + ──(existing)──────► lance-graph-contract +lance-graph-planner ──(existing)──────► lance-graph-contract + (planner does NOT dep supervisor — no cycle; verify via cargo tree) +``` + +### 5.3 The WalSink-fake honesty + +`WalSink` has **no concrete sink yet** — the concrete `LanceShardSink` is +deferred, gated on crash falsifiers (per `persistence-cycle-wal-bootstrap-v1.md` +§4). So the driver initially wires against the **same in-process fake / MemWAL +slice** the `persist_sink` probes use. This is honest and deliberate: + +> **The driver closes the CONTROL loop; the durability leg stays the +> contract-probe fake until `LanceShardSink` lands.** "Control loop closed, +> durability leg still fake" is the accurate status — `compile+test green ≠ +> storage proven` (the Ladybug rule). The P4a…e falsifiers all pass against +> the fake sink because they probe the *control* invariants (one seal, one +> version, inline fan, round-trip, watermark idempotence), none of which need +> real crash durability. Only P4f-real-durability would need the concrete +> sink, and P4f as specified is a fan-out **scale** measurement, not a +> durability claim. + +### 5.4 CognitiveWork execution is a pluggable seam (NOT designed here) + +The **thought body** — what CognitiveWork actually runs (shader / StyleStrategy +P3a/P3b) — is a **pluggable callback**, not re-specified in this plan. The +driver's job is to **FIRE the Planning→CognitiveWork step** and, after the +thought produces an Outcome, **route that Outcome through `owner_adapter` into +the next cycle's casts**. Treat thought execution as a seam: +`Fn(&Owner) -> Outcome` (or the equivalent trait object). Do **not** design the +shader here. + +--- + +## 6. 64k mechanics + the W2a scale gate + +**Scale framing** (per `mailbox-cycle-aware-write-contract-v1.md`): one basin = +one prefix table = **16k mailboxes**; **64k = ~4 basins** = the sweep target. +The fan-step (§3) is a straight pass over the fleet SoA, so 64k mailboxes = one +sync sweep of ~4 prefix tables, not 64k async operations. + +**Why `persist_sink`'s guarantees make 64k concurrent casts safe:** + +- **freeze-before-I/O** — the cycle is frozen on a detached snapshot before the + WAL append, so 64k concurrent casts collect into one immutable `Vec` + without a live mutable SoA borrow crossing the I/O. +- **one WAL write per sweep** — 64k casts amortize into a single `commit_cycle`, + so the durable-write cost is O(1) in cycles, not O(64k) in casts. +- **sealed read horizon** — every mailbox in the sweep reads exactly one sealed + predecessor `Vn`; the open cycle (`Vn+1` accumulating) is excluded, so it is + safe to read `Vn` while `Vn+1` accumulates the next 64k casts. + +**W2a scale gate (D-V3-W2a, board-as-tenant, currently GATED/deferred):** the +driver targets the **existing `MailboxSoaView::phase()` surface today** and +adopts the per-mailbox board **tenant column** (kanban board as `ValueTenant`) +when W2a un-gates. W2a is a **scale / cleanliness gate** — the fan-out becomes a +tenant *column read* instead of per-mailbox structs — **NOT a hard blocker** for +the control-loop shape. The loop closes on the `phase()` surface now; W2a makes +the 64k fan cheaper and cleaner later. This is exactly why **P4f is gated on +W2a** and labelled a scale gate, while P4a…e are not. + +--- + +## 7. Constraints / scope exclusions + +- Do **not** introduce or document **cohort internals**, participant-count + encoding, bitmap layout, or actor-neighbour firing dependencies — separate + cohort architecture work. +- Do **not** invent new **semantic, temporal, rung, witness, branch, or + ancestry** types. Reuse `KanbanMove` / `DatasetVersion` / `SweepSlot` / + `CycleFrame` / `BatchWriter` / `NextPhaseScheduler` / `KanbanActor` / + `owner_adapter` / `recover_and_apply` **verbatim**. +- The driver is a **control-loop composing existing organs** — it mints no new + subsystem. +- **Persistence stays storage-only**; the concrete `LanceShardSink` stays + **deferred** (gated on crash falsifiers). The driver wires the fake sink. +- Do **NOT** modify `persist_sink.rs` or `temporal.rs`. +- Do **not** design the CognitiveWork shader / StyleStrategy — thought execution + is a pluggable seam (§5.4). +- **Status discipline:** the driver is **PLANNED / CONJECTURE** (design), not + shipped; each claim is probe-gated, promoted to FINDING only when its falsifier + runs green. + +--- + +## 8. Status snapshot + +| Aspect | State | +|---|---| +| `persist_sink` cycle/WAL seam (`persist_cycle` / `WalSink` / `recover_and_apply`) | **SHIPPED** (D-MBX-A6-P1…P3e) — but **ZERO production callers** (the loop is open) | +| `VersionScheduler` + `NextPhaseScheduler` (sync `on_version`) | **SHIPPED** contract (D-MBX-9-IN) | +| `KanbanActor` + owner-apply (`try_advance_phase`) | **SHIPPED** (D-V3-W2b) | +| `owner_adapter` + `BatchWriter` (Outcome → next-cycle cast) | **SHIPPED** (planner) | +| `symbiont::kanban_loop::SymbiontBoard` (the shape-proving slice) | **SHIPPED** (D2) — `u32` tick placeholder for the real version | +| **CycleDriver** (P4a…f — closes seal→step→think→cast) | **PLANNED / CONJECTURE** — this plan; probe-gated | +| Home = `lance-graph-supervisor` + new planner path-dep (fallback: planner) | **DECIDED** (§5.1) — verify no cycle via `cargo tree` | +| Durability leg (concrete `LanceShardSink`, real crash durability) | **DEFERRED** — driver wires the contract-probe fake; control loop closes regardless | +| Board-as-tenant fan-out (D-V3-W2a) | **GATED** — driver uses `phase()` today; P4f scale gate adopts the tenant column when W2a un-gates | + +The organs exist; the loop does not. This plan is the record of the one seam +that makes the merged persistence bootstrap load-bearing — and of the honest +boundary that the control loop closes now while the durability leg stays a fake +until the crash falsifiers earn the concrete sink. diff --git a/.claude/plans/persistence-cycle-wal-bootstrap-v1.md b/.claude/plans/persistence-cycle-wal-bootstrap-v1.md index 100eb94c..b3dc3aa5 100644 --- a/.claude/plans/persistence-cycle-wal-bootstrap-v1.md +++ b/.claude/plans/persistence-cycle-wal-bootstrap-v1.md @@ -1,6 +1,8 @@ # persistence-cycle-wal-bootstrap-v1 — the primitive cycle/WAL seam and its temporal/revision upgrade path -> **Status:** ACTIVE (bootstrap SHIPPED in PR #878; upgrade phases PLANNED). +> **Status:** ACTIVE (bootstrap SHIPPED in PR #878; upgrade phases PLANNED; the +> §2 sparse-delta storage rule is RATIFIED architecture, UNIMPLEMENTED in a +> concrete Lance sink). > **Date:** 2026-08-02. > **Scope:** documentation-only architectural ruling. Records the *role* of the > #878 persistence seam and the intended larger two-dimensional temporal @@ -60,12 +62,149 @@ The current scalar slot + single-key ordering model is **sufficient** to establish the execution and durability plumbing, and **nothing more**. It is **not** claimed to be the final representation of temporal or cognitive order. This document does **not** redesign or fix that limitation — recording the -boundary is the whole point. The upgrade path is §2–§3; the accepted debts are -§4. +boundary is the whole point. The sparse-delta storage ruling is §2; the +two-dimensional upgrade path is §3–§4; the accepted debts are §5. --- -## 2. The intended larger architecture — two orthogonal dimensions +## 2. A complete logical cycle is physically SPARSE (RATIFIED architecture, UNIMPLEMENTED in a concrete sink) + +> **Status of this section:** the sparse-delta rule is **RATIFIED as +> architecture** and **UNIMPLEMENTED in a concrete Lance sink**. The #878 +> bootstrap remains SHIPPED; this section governs the *future* concrete sink, +> not the merged contract-probe. + +**"One complete cycle image" must NEVER be read as serializing every row merely +because every participant belonged to the cycle.** The load-bearing distinction: + +``` +complete logical cycle ≠ full physical dataset rewrite +``` + +A cycle is **logically complete** when: + +- all required participants reached the cycle boundary, +- all produced updates were collected, +- updates were temporally ordered / coalesced, +- all required lifecycle transitions were included, +- the resulting change set was frozen, +- the change set was committed atomically. + +The **physical payload stays sparse**: + +``` +64k participants + → N dirty rows (N may be ≪ 64k) + → one frozen sparse delta batch + → one WAL transaction + → one DatasetVersion +``` + +Unchanged rows are **inherited from the sealed predecessor version** and MUST +NOT be serialized merely because they participated in the cycle. + +### The storage invariant (RATIFIED) + +> **«A cycle is globally complete but physically sparse. `commit_cycle` +> persists only the coalesced dirty-row set and the required durable transition +> metadata. Unchanged rows remain inherited from the sealed base version and do +> not become new row payloads merely because they were members of the cycle.»** + +**"One WAL write per cycle" means one atomic durability boundary for the sparse +change set.** It does **not** mean one full 64k-row (~32 MiB) snapshot per +cycle. + +### Participation is separate from mutation + +Cycle participation / completion evidence is represented **compactly and +separately** from row payloads: + +``` +cycle completion evidence (small footer): + cycle identity + sealed base version + expected / completed participation evidence or digest + dirty-row count + transition count + batch digest + +physical delta: + only rows whose FINAL state changed + only the required durable lifecycle transitions +``` + +**A participant that produced no state mutation MUST NOT require a 512-byte row +payload merely to prove participation.** (Cohort internals — participant-count +encoding, bitmap layout, ownership — are out of scope here; they belong to the +cohort architecture session.) + +### Payload-duplication warning (honest bootstrap limitation) + +The #878 contract-probe shape currently duplicates bytes in memory: + +- `SweepSlot` owns payload bytes, +- `DetachedCycleBatch` retains the per-landing records, +- `freeze` also clones the final row payloads into the coalesced `image`. + +This is acceptable for the in-memory contract-probe fake. **The concrete Lance +sink must NOT persist duplicate copies of both the per-landing payload bytes AND +the final coalesced row image.** The future concrete shape keeps the three +concerns distinct: + +``` +landing metadata / durable transitions + + one detached coalesced dirty-row image + + a small cycle footer / completion evidence +``` + +*(Recorded as a concrete-sink upgrade requirement — the Rust structs are NOT +redesigned in this task.)* + +### Capacity + backpressure (concrete-sink ruling) + +``` +normal cycle: a sparse dirty-row delta +worst case: every row genuinely dirty → ≈ one full row slab +``` + +A genuinely dense cycle is **valid**, but it is an **explicit capacity event**, +not the default storage shape. The future concrete sink MUST define (numeric +thresholds are NOT chosen in this documentation task): + +- maximum frozen cycles in flight, +- maximum bytes in flight, +- backpressure when WAL / storage falls behind, +- checkpoint / compaction policy, +- version-retention policy, +- disk-space monitoring + a refusal threshold. + +### Required future falsifiers (concrete-sink, probe-first) + +1. **Sparse-cycle** — 64k logical participants, 17 dirty rows → one WAL + transaction → **exactly 17** coalesced row payloads written → one + `DatasetVersion` → unchanged rows inherited from `Vn`. +2. **No-op-cycle policy** — zero dirty rows AND zero durable transitions → the + sink follows ONE explicitly documented policy: *either* no new + `DatasetVersion`, *or* a metadata-only cycle version. The policy is chosen + before the concrete sink ships; it MUST never write a full empty row slab. +3. **Coalescing** — many updates to the same row in one cycle → **one** final + row payload physically written; intermediate payload copies are NOT persisted + as duplicate row state; the required transition history remains available + separately. +4. **Dense-cycle capacity** — all rows genuinely dirty → one bounded dense + batch; backpressure prevents an unbounded queue of dense frozen cycles; no + silent disk exhaustion. +5. **Retention** — many `DatasetVersion`s accumulate → a documented + retention / checkpoint policy bounds disk growth; versions inside the + configured hindsight horizon remain readable. + +**Scope fence for this section:** it concerns *physical persistence density +only*. It does not pull cohort topology (§6), horizontal partial ordering +(§3.1), or revision semantics (§4) into the concrete-sink design. + +--- + +## 3. The intended larger architecture — two orthogonal dimensions The final architecture is two **orthogonal** dimensions. #878 supplies the vertical axis primitively and leaves the horizontal axis to a later phase. @@ -127,11 +266,11 @@ durable states — the lookup is over already-coherent frames. > The version table performs **vertical** frame succession and lookup **only**. > It does **not** perform horizontal causal ordering — that is the horizontal -> dimension's job (§2.1). Conflating the two is the error this section forecloses. +> dimension's job (§3.1). Conflating the two is the error this section forecloses. --- -## 3. Planned temporal error-correction phase (later; not #878) +## 4. Planned temporal error-correction phase (later; not #878) The primitive #878 slot model may initially produce frames whose **horizontal coherence is incomplete** — the scalar order captures *that* results happened, @@ -170,33 +309,37 @@ Lance versions preserve the successive durable vertical frames not be silently rewritten** merely because a later metacognitive pass revised its interpretation. Correction flows *forward* — into a later cycle / later version — never *backward* over a sealed frame. (This is the vertical-axis -immutability that keeps the hindsight-lookup in §2.2 honest.) +immutability that keeps the hindsight-lookup in §3.2 honest.) --- -## 4. Known limitations accepted for #878 +## 5. Known limitations accepted for #878 Recorded explicitly as **upgrade points**, not as claims that #878 already solves the final temporal model: - The scalar slot model is **provisional**. -- A scalar key **may not capture** future partial-order cognition (§2.1). +- A scalar key **may not capture** future partial-order cognition (§3.1). - Cross-owner and equal-position **conflict semantics are not finalized**. - Cycle **retry** and **production WAL idempotence** still require concrete-sink hardening. - The **fake WAL sink proves the contract shape, not real crash durability** (`compile+test green ≠ storage proven`, the Ladybug lesson). -- The **complete horizontal temporal projection remains future work** (§2.1, +- The contract-probe shape **duplicates payload bytes in memory** (`SweepSlot` + + `DetachedCycleBatch` landings + the coalesced `image`); the concrete sink must + not persist both copies. Full statement + the sparse-delta storage invariant + and the five concrete-sink falsifiers are in §2. +- The **complete horizontal temporal projection remains future work** (§3.1, cross-ref `temporal-markov-and-style-classes-v1.md`). --- -## 5. Scope exclusions (this document and the #878 bootstrap) +## 6. Scope exclusions (this document and the #878 bootstrap) - Do **not** introduce or document detailed **cohort internals**. - Do **not** mention a **fixed number of cohort slots**. - Do **not** design **actor-neighbour waiting or firing dependencies** (the - emit path is wait-free by §3). + emit path is wait-free by §4). - Do **not** invent new **semantic, temporal, rung, witness, branch, or ancestry** types. - Do **not** revive `ThoughtWitness`, `basis`, `awareness_seq`, or **per-cast @@ -207,16 +350,17 @@ solves the final temporal model: --- -## 6. Status snapshot +## 7. Status snapshot | Aspect | State | |---|---| | Cycle/WAL bootstrap seam (`persist_sink`) | **SHIPPED** in PR #878 (bootstrap; contract-probed, not storage-proven) | | Vertical axis (`DatasetVersion` succession + lookup) | Established primitively by the seam | | Horizontal axis (`temporal.rs` coherence over a frame) | **PLANNED** — owned by `temporal-markov-and-style-classes-v1.md` | -| Shadow temporal-coherence correction pass | **PLANNED** (§3) | -| `revision.rs` forward-correction mechanism | **PLANNED** (§3) | -| Concrete Lance sink (real crash durability) | **DEFERRED** — gated on crash falsifiers (§4) | +| Shadow temporal-coherence correction pass | **PLANNED** (§4) | +| `revision.rs` forward-correction mechanism | **PLANNED** (§4) | +| Concrete Lance sink (real crash durability) | **DEFERRED** — gated on crash falsifiers (§5) | +| Sparse-delta storage rule (complete cycle ≠ full rewrite) | **RATIFIED architecture, UNIMPLEMENTED in a concrete sink** (§2) | The bootstrap exists so the rest can be built on a running, durable seam. The scalar order is a load-bearing placeholder, and this document is the record that From d74520c3bf6b8e50c1602eb3c8e0ed2d4376feea Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 10:03:29 +0000 Subject: [PATCH 2/6] docs(arch): ratify D-MBX production spine + correct P4 to sparse sealed-transition application MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architecture/plan reconciliation before implementing the cycle-loop closure. Documentation-only — no Rust code, tests, public APIs, or persistence implementation changed (verified: zero .rs in changeset). Crate deps verified against Cargo.toml, not asserted from memory. 1) THE CORRECTION (ChatGPT feedback, accepted — it caught a real same-session contradiction). The loop-closure plan's P4 had the supervisor FAN NextPhaseScheduler::on_version across the whole fleet → advance every non-absorbing mailbox per sealed version. That makes almost the entire fleet dirty every cycle, directly violating the sparse-cycle ruling (E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1). Corrected model throughout §2/§3/§4/§6: a DatasetVersion is GLOBAL KNOWLEDGE, not permission to advance every mailbox. Owners think over Vn; owners that produce material updates emit sparse fire-and-forget intents; planner collects/coalesces/seals (one WAL → Vn+1) and exposes the sealed paired-transition set; the supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical), inline (no dataset re-read; NOT 64k async drive_once). on_version becomes the intent-time lowering policy, not an apply-time fan; SymbiontBoard.step-advances-every-board is the SLICE shape-prover, not the production rule. Interim rule: <=1 durable phase transition per owner per sealed cycle. P4b falsifier is now the sparse shape: 64k mailboxes / 17 sealed transitions -> exactly 17 advance, rest byte-identical, no second dataset read, one version. P4a..P4f re-scoped to supervisor-side sparse application. 2) RATIFIED OWNERSHIP MAP (§9, verified deps): contract = canonical types (zero fleet ownership); cognitive-shader-driver = MailboxSoA type/layout home + anatomy (not the runtime lifecycle); planner = decides + persistence contract (never mutates a supervisor-owned SoA, never deps supervisor); supervisor = exclusive runtime owner + P4 loop + applies only sealed sparse transitions; lance = storage substrate + external-reader subscription + future LanceShardSink. Dep direction verified: supervisor->contract, planner->contract, planner does NOT dep supervisor/symbiont/rs-graph-llm; supervisor->planner is the planned acyclic P4 edge; shader has an optional feature-gated planner dep (debug DTOs, not fleet ownership). NO cycles found. 3) ADJACENT-CRATES DOCTRINE (§10): symbiont = golden-image + bystander research lab (forbidden as authoritative owner/scheduler/WAL/version/required dep); rs-graph-llm = optional capability basement (client/capability provider returning Outcomes, never owns the standing wave); ogar-* = AST/declaration/ adapter basement (describes behaviour, never owns the cycle). Subagent anti-drift guardrail with STOP+report triggers (§11). 4) DRIFT AUDIT: the only genuine conflict was the P4 fan-step introduced this session (fixed). Other 'fan'/symbiont hits are corrected text, append-only AGENT_LOG history, or SymbiontBoard-slice / ractor-compile-time-argument descriptions in sibling plans (not production-ownership claims; now governed by the §10 doctrine). Board: EPIPHANIES E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1 (prepend); INTEGRATION_PLANS + STATUS_BOARD P4 row corrected to sparse. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 12 + .claude/board/INTEGRATION_PLANS.md | 47 +-- .claude/board/STATUS_BOARD.md | 2 +- .claude/plans/cycle-loop-closure-driver-v1.md | 297 ++++++++++++++---- 4 files changed, 277 insertions(+), 81 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 92206823..2192e4ad 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,15 @@ +## 2026-08-02 — E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1 — the D-MBX production ownership map is ratified, and P4's fleet-wide fan-step is corrected to sparse sealed-transition application + +**Status:** RATIFIED (operator ruling, 2026-08-02) + CORRECTION of a same-session drift. **Confidence:** High for the ownership map (verified against `Cargo.toml` deps) and the sparse correction (it removes a direct contradiction with `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). Documentation-only — no Rust/tests changed. Recorded in `.claude/plans/cycle-loop-closure-driver-v1.md` §3, §9–§11. + +**The production spine (straight railway track).** contract **defines** · planner **proposes and seals** · supervisor **owns and applies** · shader **thinks** · Lance **persists**. Ownership map (verified deps): `lance-graph-contract` = canonical types (KanbanColumn/KanbanMove, DatasetVersion, VersionScheduler traits, MailboxSoaView/Owner, legal Rubicon transitions; zero-dep; no fleet ownership, no persistence impl, no thought body). `cognitive-shader-driver` = **MailboxSoA type/layout home** + native cognition/shader/thinking (defines the anatomy; NOT the runtime fleet lifecycle). `lance-graph-planner` = **decides** (StyleStrategy/StrategyOutcome, intended KanbanMove, owner_adapter, BatchWriter, cycle collection+coalescing, persist_cycle/WalSink contract, recovery/temporal contracts; **never mutates a supervisor-owned MailboxSoA**). `lance-graph-supervisor` = **exclusive runtime owner** (KanbanActor state IS the owner; authoritative phase mutation; the P4 cycle-loop; applies only sealed sparse transitions; fires CognitiveWork; returns Outcomes; **owns D-MBX-A6-P4**). `lance-graph` = Lance dataset + DatasetVersion substrate + external-reader subscription + future concrete LanceShardSink (storage, NOT cognitive fleet owner). The three-way split that must stay explicit: **MailboxSoA type = shader; runtime ownership = supervisor; decision + persistence-contract = planner.** + +**Dependency direction (verified 2026-08-02).** `supervisor → contract` (+callcenter) and `planner → contract` exist; `planner` does NOT dep supervisor/symbiont/rs-graph-llm; `contract` is zero-dep; `cognitive-shader-driver` has an OPTIONAL feature-gated planner dep (debug/serve DTOs, not fleet ownership). The `supervisor → planner` edge is the **planned P4 wiring** and is acyclic (planner's closure never reaches supervisor). **No dependency cycles found.** + +**The correction — a version is NOT a fleet-step signal.** The earlier same-session draft of the P4 loop had the supervisor **fan `NextPhaseScheduler::on_version` across the whole fleet → advance every non-absorbing mailbox** on each sealed version. That is WRONG: it makes almost the entire fleet dirty every cycle, directly violating the sparse-cycle ruling. **Corrected model:** mailboxes think over the sealed `Vn`; owners that produce material updates emit sparse fire-and-forget intents (a `SweepSlot` with a `paired_move`); planner collects/coalesces/seals (one WAL → `Vn+1`) and exposes the **sealed paired-transition set**; the supervisor iterates **ONLY those sealed transitions**, resolves each owner, applies one legal `try_advance_phase`, and leaves **all unrepresented owners byte-identical**. `NextPhaseScheduler::on_version` is the **intent-time lowering policy**, never an apply-time fan. `SymbiontBoard.step`-advances-every-board is the SLICE shape-prover, not the production rule. **Canonical distinction: `DatasetVersion` is global knowledge; Kanban mutation is sparse and owner-specific; a new version is never, by itself, permission to advance every mailbox.** P4b's falsifier is now the sparse shape: 64k mailboxes / 17 sealed transitions → exactly 17 advance, rest byte-identical, no second dataset read, one version. **Interim conservative rule:** ≤1 durable Kanban phase transition per owner per sealed cycle (data updates coalesce; extra state-dependent transitions wait for the next sealed horizon). The writer applies the sealed set INLINE (no dataset re-read; NOT 64k async `drive_once`). + +**Adjacent-crates doctrine (basements, not owners).** `symbiont` = golden-image + bystander research lab (compile/link golden image, integration/scale probes, AST-arm + arm-discovery experiments) — FORBIDDEN as authoritative MailboxSoA owner / production scheduler / Kanban lifecycle / WAL owner / version authority / required dep of the core crates. `rs-graph-llm` = optional capability basement (agentic demos, Rig, tool-use adapters, HITL façades, optional CognitiveWork capability providers) — FORBIDDEN as authoritative SoA/Kanban/WAL/version state or a required dep; it is a client/capability provider that returns Outcomes/evidence, never owns the standing wave. `ogar-*` = AST/declaration/adapter basement — describes behaviour, never owns the living cycle. Subagent guardrail (STOP+report triggers) recorded in the plan §11. Extends `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1`, `E-SUBSTRATE-IS-THE-SCHEDULER`, `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`. + ## 2026-08-02 — E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1 — a globally-complete cycle persists only its coalesced dirty-row delta, never a full 64k-row snapshot **Status:** RATIFIED as architecture; UNIMPLEMENTED in a concrete Lance sink (operator ruling, 2026-08-02). **Confidence:** High for the ruling (a storage-density invariant); the concrete sink that must honor it is deferred and unbuilt. Documentation-only — no Rust/tests changed. Recorded in `.claude/plans/persistence-cycle-wal-bootstrap-v1.md` §2. diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index de716fcf..60cafbab 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -4,25 +4,34 @@ The loop-closure driver: the missing seam that turns the merged `persist_sink` cycle/WAL bootstrap into a running loop at 64k concurrency. Today `persist_sink::{persist_cycle, WalSink, versions}` has **zero production -callers** — the loop is open. The driver closes `collect casts → persist_cycle → -sealed DatasetVersion → sync fan-step across the mailbox fleet → -try_advance_phase (the KanbanStep) → CognitiveWork runs the thought → -owner_adapter casts the next intent → back to collect`. Correctness pivot: the -driver WROTE the version, so it fires `NextPhaseScheduler::on_version` + -`try_advance_phase` **inline and synchronously** (no dataset re-read) — NOT 64k -async `LanceVersionScheduler::drive_once` (that subscription variant is for -reading a version you did NOT write). Mints NO new types — composes -`KanbanMove`/`DatasetVersion`/`SweepSlot`/`BatchWriter`/`NextPhaseScheduler`/ -`KanbanActor`/`owner_adapter`/`recover_and_apply`. Deliverables D-MBX-A6-P4a -(driver skeleton) → P4b (fleet fan-step) → P4c (loop closure round-trip) → P4d -(wait-free-emit guard) → P4e (recovery composition) → P4f (16k/64k scale, -W2a-gated), each probe-first. Home: `lance-graph-supervisor` (structural fleet -owner; new planner path-dep, no cycle) with a planner fallback. HONEST: the -CONTROL loop closes; the durability leg stays the contract-probe fake until the -concrete `LanceShardSink` lands. Board-as-tenant (D-V3-W2a) is a SCALE gate, not -a control-loop blocker. Companion to `persistence-cycle-wal-bootstrap-v1.md` -(which also gained the §2 sparse-delta storage ruling this session — see -EPIPHANIES `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). +callers** — the loop is open. The driver closes: owners think over the sealed +`Vn`; owners that produce material updates emit **sparse** fire-and-forget +intents; the planner collects/coalesces/freezes one cycle (one WAL, `Vn+1`) and +exposes the **sealed paired-transition set**; the supervisor applies **ONLY the +sealed sparse transitions** (each represented owner advances one legal step; +**all unrepresented owners stay byte-identical**); owners entering CognitiveWork +run the thought and cast the next intent via `owner_adapter`. **Correctness +pivot (corrects the earlier draft): a `DatasetVersion` is global knowledge, NOT +permission to advance every mailbox** — the earlier "fan `on_version` across the +whole fleet" model violated the sparse-cycle ruling and is removed. The sealed +transitions are applied INLINE by the writer (no dataset re-read; NOT 64k async +`LanceVersionScheduler::drive_once`, which is the reader-that-did-not-write +variant). Interim rule: ≤1 durable phase transition per owner per sealed cycle. +Mints NO new types — composes `KanbanMove`/`DatasetVersion`/`SweepSlot`/ +`BatchWriter`/`NextPhaseScheduler`/`KanbanActor`/`owner_adapter`/ +`recover_and_apply`. Deliverables D-MBX-A6-P4a (drain+seal) → P4b (apply sealed +sparse set; falsifier: 64k mailboxes / 17 sealed transitions → exactly 17 +advance, rest byte-identical) → P4c (CognitiveWork+cast round-trip) → P4d +(wait-free emit) → P4e (recovery composition) → P4f (sparse-routing scale +16k/64k, W2a-gated), each probe-first. Home: `lance-graph-supervisor` (structural +fleet owner; new one-way planner path-dep, verified acyclic) with a planner +fallback. Also carries the D-MBX crate-responsibility map (§9), the +adjacent-crates doctrine for symbiont / rs-graph-llm / ogar-* (§10), and the +subagent anti-drift guardrail (§11). HONEST: the CONTROL loop closes; the +durability leg stays the contract-probe fake until the concrete `LanceShardSink` +lands. Board-as-tenant (D-V3-W2a) is a SCALE gate, not a control-loop blocker. +Companion to `persistence-cycle-wal-bootstrap-v1.md` §2 sparse-delta ruling +(EPIPHANIES `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). ## 2026-08-01 — CORRECTION to the §8 entry below: `RungLevel 0–10` → `0–9` diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index f873555e..53033e01 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -964,7 +964,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | | D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **Merged** | #878 (merged; reshaped in place to the cycle/WAL model = P3e — `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`; + the §2 sparse-delta storage ruling `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`, RATIFIED/UNIMPLEMENTED); ordering/recovery CONTRACT probed (348 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | -| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `collect casts → persist_cycle → sealed DatasetVersion → sync fan-step (`NextPhaseScheduler::on_version` + `try_advance_phase`, inline — NOT 64k async `drive_once`) → CognitiveWork → `owner_adapter` casts next intent → loop`. Mints NO new types. Sub-deliverables P4a (driver skeleton) / P4b (fleet fan-step) / P4c (round-trip closure) / P4d (wait-free-emit) / P4e (recovery) / P4f (16k/64k scale, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **Queued (PLANNED)** | plan `.claude/plans/cycle-loop-closure-driver-v1.md`; home lance-graph-supervisor (new planner dep, no cycle; planner fallback); CONTROL loop closes, durability leg stays the contract-probe fake until concrete `LanceShardSink`; consumes #878 persist_sink; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | +| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **Queued (PLANNED)** | plan `.claude/plans/cycle-loop-closure-driver-v1.md`; home lance-graph-supervisor (new one-way planner dep, verified acyclic; planner fallback); CONTROL loop closes, durability leg stays the contract-probe fake until concrete `LanceShardSink`; consumes #878 persist_sink; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/.claude/plans/cycle-loop-closure-driver-v1.md b/.claude/plans/cycle-loop-closure-driver-v1.md index ae5b4c57..8e4d6715 100644 --- a/.claude/plans/cycle-loop-closure-driver-v1.md +++ b/.claude/plans/cycle-loop-closure-driver-v1.md @@ -12,8 +12,10 @@ > falsifiers that gate its construction. Changes **no** Rust code, tests, > public APIs, `persist_sink.rs`, or `temporal.rs`. > **Owns (narrowly):** "why the loop is open today", the closed-loop -> seal→step→think→cast shape, the writer-fires-inline correctness point, the -> D-MBX-A6-P4a…f deliverables, the home + dep-direction decision, and the 64k +> seal→step→think→cast shape, the SPARSE sealed-transition application rule + +> the writer-fires-inline correctness point, the D-MBX-A6-P4a…f deliverables, +> the home + dep-direction decision, the D-MBX crate-responsibility map (§9), +> the adjacent-crates doctrine (§10), the subagent guardrail (§11), and the 64k > mechanics as they bear on the control loop. > **Does NOT own (cross-refs, never re-specifies):** > - The cycle/WAL seam itself (the OUT/durability half + two-dimensional @@ -46,7 +48,7 @@ The organs all exist and are merged. The loop is **not** closed. this plan, **zero production callers** (verified by grep). It is a load-bearing seam with nothing standing on it. The cycle can be frozen, one WAL write can be made, one sealed `DatasetVersion` can be returned — but *nothing calls it in a -running loop*, so the version it seals never fans out to advance any mailbox, +running loop*, so the version it seals never triggers any sealed owner's step, and no finished thought ever casts the next cycle's intent. Everything the loop needs is already built, in five separate crates: @@ -96,17 +98,17 @@ The driver closes exactly this loop: persist_cycle(sink, CycleFrame{cycle, base_version=Vn}, casts) │ (one WAL write, freeze-before-I/O) ▼ - sealed DatasetVersion Vn+1 - │ - ▼ - fan the step across the mailbox fleet ← writer fires INLINE (§3) - NextPhaseScheduler::on_version(view_i, Vn+1, exec) (sync, pure) - │ + sealed DatasetVersion Vn+1 + the sealed PAIRED-TRANSITION SET + │ (only the cycle's SweepSlots carrying a paired_move — + │ a SPARSE subset, NOT the whole fleet) ▼ - try_advance_phase per mailbox ← the KanbanStep (KanbanActor) + supervisor iterates ONLY the sealed paired transitions ← writer fires INLINE (§3) + for each sealed (owner, paired_move): + resolve owner → try_advance_phase(paired_move.to) ← the KanbanStep (KanbanActor) + all UNREPRESENTED owners remain BYTE-IDENTICAL (untouched) │ ▼ - CognitiveWork runs the thought ← pluggable callback (§5.4 seam) + owners entering CognitiveWork run the thought ← pluggable callback (§5.4 seam) │ produces an Outcome ▼ owner_adapter: Outcome → emit_bootstrap_intent → BatchWriter::cast(on_behalf=owner) @@ -128,43 +130,82 @@ exists. --- -## 3. Correctness — the writer fires the step INLINE and SYNCHRONOUSLY (not 64k `drive_once`) - -This is the load-bearing correctness point. - -**The driver WROTE the version** — `persist_cycle` returned `Vn+1`. Because the -driver already holds the version it committed, it fires the fan-step **inline -and synchronously**: - -``` -NextPhaseScheduler::on_version(view_i, Vn+1, exec) // sync pure fn, per mailbox - → try_advance_phase(mailbox_i) // per mailbox -``` - -`on_version` is a **sync pure function** (board D2 line: the writer knows the -version it committed and fires the update inline, no async). The sweep is a -straight-line pass over the mailbox SoA — one sync call per mailbox, then the -per-mailbox `try_advance_phase`. There is **no second dataset read** between the -seal and the step. - -The driver does **NOT** fan 64k async `drive_once` / `drive_at_latest` calls. +## 3. Correctness — SPARSE sealed-transition application, fired INLINE (a version is NOT permission to advance every mailbox) + +Two load-bearing correctness points. The first is the one the earlier draft of +this plan got **wrong** and this revision fixes. + +### 3.1 A new version is global knowledge; Kanban mutation is SPARSE and owner-specific + +> **`DatasetVersion` is global knowledge. Kanban mutation is sparse and +> owner-specific. A new version is NEVER, by itself, permission to advance every +> mailbox.** + +The **rejected** model (the earlier draft, corrected here): "sealed +`DatasetVersion` → fan `NextPhaseScheduler::on_version` across the fleet → +advance every non-absorbing mailbox." That is wrong — it makes almost the entire +fleet dirty every cycle (every mailbox's phase changes), which **violates the +sparse-cycle ruling** (`persistence-cycle-wal-bootstrap-v1.md` §2 / +`E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). A global clock +tick is not a fleet-wide step signal. + +The **correct** production loop: + +- **Mailboxes think concurrently over the sealed `Vn`.** +- **Owners that produce material updates emit fire-and-forget intent** — a + `SweepSlot` carrying a `paired_move` — through `BatchWriter::cast` (via + `owner_adapter`). The intended move is decided at **intent time** by the + planner (StyleStrategy / `owner_adapter`, optionally using the + `NextPhaseScheduler` forward-arc as the lowering policy). It is *proposed*, not + yet authoritative. +- **Planner** collects + coalesces the sparse casts, freezes one cycle, performs + **one WAL transaction**, receives the sealed `DatasetVersion Vn+1`, and exposes + the **sealed paired-transition set** (the SweepSlots that carry a `paired_move`). +- **Supervisor** iterates **only the sealed paired transitions**, resolves the + corresponding owner for each, and applies **one legal transition** to each + **represented** owner via `try_advance_phase(paired_move.to)`. **All + unrepresented owners remain byte-identical** — they are not touched, not + re-serialized, not swept. +- **Owners entering CognitiveWork** run the native thought body, produce + Outcomes, and route them through `owner_adapter` into the next cycle. + +So the phase mutation is driven by the owner **having produced a sealed intent**, +never by the version tick fanning to everyone. `NextPhaseScheduler::on_version` +is the intent-time **lowering policy** (which move is legal for an owner that +produced an update), *not* an apply-time fan across the fleet. The symbiont +`SymbiontBoard.step` (D2) advances every board per tick — that is the SLICE +shape-prover, **not** the production apply rule. + +**Interim conservative rule (record it):** *at most one durable Kanban phase +transition per owner per sealed cycle.* Multiple data updates for the same owner +may coalesce (per-row, per `persist_sink`), but additional **state-dependent +phase transitions wait for the next sealed horizon** — an owner advances at most +one Rubicon edge per seal. + +### 3.2 The writer fires the sealed transitions INLINE and SYNCHRONOUSLY (not 64k `drive_once`) + +**The supervisor/driver WROTE the version** — `persist_cycle` returned `Vn+1` +and the sealed paired-transition set is already in hand. So it applies those +transitions **inline and synchronously** — a straight iteration over the sparse +sealed set, one `try_advance_phase` per represented owner. There is **no second +dataset read** between the seal and the step. + +The driver does **NOT** fan async `drive_once` / `drive_at_latest` calls. `LanceVersionScheduler::drive_once` / `drive_at_latest` are the **subscription** variant — async precisely because they READ a version they did NOT write, and -**each opens the Lance dataset**. Fanning 64k of those across a fleet would be -64k dataset opens to re-read a version the driver already has in hand: wrong, -and quadratically wrong at scale. - -The rule, stated for the record: +**each opens the Lance dataset**. Using them here would re-read a version the +driver already holds; across a fleet that is 64k dataset opens for nothing. -> **Async is ONLY (a) the `persist_cycle` I/O leg and (b) the subscription -> drive path (a reader that did not write the version). The writer-side -> fan-out is sync `on_version` + `try_advance_phase`, inline, no dataset +> **Async is ONLY (a) the `persist_cycle` I/O leg and (b) the subscription drive +> path (a reader that did not write the version). The writer-side application of +> the sealed sparse transitions is inline `try_advance_phase`, no dataset > re-read.** The subscription path (`drive_at_latest`) remains the correct tool for a -*separate* reader process that observes sealed versions it did not produce — -that reader legitimately opens the dataset because it has no other handle to the -version. That is a different actor from the driver and out of scope here. +*separate external reader* that observes sealed versions it did not produce (it +legitimately opens the dataset because it has no other handle to the version). +That reader is a different actor from the driver and out of scope here +(D-MBX-9-IN external-reader implementation, `lance-graph`). --- @@ -176,16 +217,16 @@ silent twins are called out where the naive assertion would be vacuous. | ID | Deliverable | Falsifier (what input makes it fail) | |---|---|---| -| **D-MBX-A6-P4a** | **CycleDriver skeleton** — drain the fleet's `BatchWriter` casts into `Vec` → `persist_cycle` → sealed `DatasetVersion`. | N staged casts produce **exactly one** WAL write + **exactly one** version (reuse `persist_sink`'s amortization probe at the driver level — assert `commit_cycle` invoked once, not N times). | -| **D-MBX-A6-P4b** | **fleet fan-step** — sealed version → sync `on_version` sweep over the mailbox SoA → `try_advance_phase` per mailbox (via `KanbanActor` / owned sweep). | (i) **anti-vacuity:** a fleet with a *mix* of phases — each mailbox advances by exactly one **legal forward-arc** step AND assert different mailboxes took **different** steps (not lockstep-blind); (ii) an **absorbing** mailbox fires nothing (`on_version` → `None`); (iii) assert **NO second dataset read** between seal and step (proves the writer fires inline per §3, not via a re-read `drive_once`). | -| **D-MBX-A6-P4c** | **loop closure** — a CognitiveWork Outcome → `owner_adapter::emit_bootstrap_intent` → `BatchWriter::cast` into the NEXT cycle → appears in Vn+1's collected casts (round-trip). | An Outcome cast in cycle N is **present in cycle N+1's collected casts** AND advances the owner **one step further** (not merely enqueued — actually collected and applied next cycle). | -| **D-MBX-A6-P4d** | **wait-free-emit guard** — a mailbox whose neighbour has NOT completed still advances (no synchronous neighbour wait). | **can-it-fire:** construct a fleet where mailbox B is mid-thought and mailbox A completes — A **still steps in the same cycle**; assert **no barrier** / no neighbour wait blocked A. | -| **D-MBX-A6-P4e** | **recovery composition** — `recover_and_apply` replays the owner's pending tail after a mid-loop stop, idempotent with the watermark. | Stop mid-loop, re-drive, assert **no double-apply** (reuse the `persist_sink` watermark probe at the driver level — `applied_through` gates the replay so a re-applied slot is a no-op). | -| **D-MBX-A6-P4f** *(SCALE, gated on W2a)* | **16k / 64k mailboxes fan in one cycle** within the cycle budget. | **MEASURED**, labelled a **scale gate, not a correctness claim**: 16k/64k mailboxes fan in one cycle within the ~0.5–2.5 s/cycle budget; **log what was measured, never a silent cap**. | +| **D-MBX-A6-P4a** | **supervisor drains planner casts and calls `persist_cycle`** — collect the fleet's staged `BatchWriter` casts into `Vec` → `persist_cycle` → sealed `DatasetVersion` + the sealed paired-transition set. | N staged casts produce **exactly one** WAL write + **exactly one** version (reuse `persist_sink`'s amortization probe at the driver level — assert `commit_cycle` invoked once, not N times). | +| **D-MBX-A6-P4b** | **supervisor applies ONLY the sealed sparse transition set** — iterate the cycle's sealed `paired_move` SweepSlots, resolve each owner, apply one legal `try_advance_phase`; leave every unrepresented owner byte-identical. | **The sparse falsifier:** 64k registered mailboxes; **17** owners have sealed paired transitions → **exactly those 17 owners advance** → **all other owner rows remain byte-identical** → **no second dataset read** → **one `DatasetVersion`**. (Anti-vacuity: assert the untouched set is the other 64k−17, not merely that 17 advanced; assert a mix of legal edges, not lockstep.) | +| **D-MBX-A6-P4c** | **owners entering CognitiveWork run the thought and cast the next intent** — the represented owner runs the pluggable thought body, produces an Outcome, routes it via `owner_adapter::emit_bootstrap_intent` → `BatchWriter::cast` into the NEXT cycle. | An Outcome cast in cycle N is **present in cycle N+1's collected casts** AND, when N+1 seals, advances that owner **one further legal step** (round-trip: not merely enqueued — collected and applied next cycle). | +| **D-MBX-A6-P4d** | **one completed owner never waits synchronously for an unrelated owner** — the emit path is wait-free; a finished owner casts + advances without blocking on a neighbour. | **can-it-fire:** a fleet where owner B is mid-thought and owner A completes — A **still emits + (if sealed) advances in the same cycle**; assert **no barrier / no neighbour wait** blocked A. | +| **D-MBX-A6-P4e** | **supervisor composes planner recovery, applies only unreplayed moves** — on a mid-loop restart, `recover_and_apply` replays the owner's pending tail, idempotent with the durable watermark. | Stop mid-loop, re-drive, assert **no double-apply** (reuse the `persist_sink` watermark probe — `applied_through` gates the replay so an already-applied slot is a no-op; represented owners advance once, unrepresented untouched). | +| **D-MBX-A6-P4f** *(SCALE, gated on W2a)* | **measure sparse routing + cycle cost at 16k / 64k** — the cost of resolving + applying the sealed sparse set (NOT a full sweep) at fleet scale. | **MEASURED**, labelled a **scale gate, not a correctness claim**: at 16k/64k registered mailboxes with a realistic sparse dirty fraction, measure sealed-set routing + apply within the ~0.5–2.5 s/cycle budget; **log the dirty fraction + what was measured, never a silent cap**. | -**Sequencing:** P4a (collect+seal) and P4b (fan-step) are the spine; P4c closes -the round-trip; P4d and P4e are the wait-free + recovery guards on the spine; -P4f is the scale gate, deferred with W2a (§6). +**Sequencing:** P4a (collect+seal) and P4b (apply the sparse sealed set) are the +spine; P4c closes the round-trip; P4d and P4e are the wait-free + recovery guards +on the spine; P4f is the sparse-routing scale gate, deferred with W2a (§6). --- @@ -197,7 +238,8 @@ P4f is the scale gate, deferred with W2a (§6). fleet owner. It already owns `KanbanActor` + the owner-apply surface (`try_advance_phase`, `drive_version_tick`, `drive_scheduled_tick`), which is exactly the "apply" leg of the loop. Putting the control-loop next to the apply -surface keeps the fan-step where the fleet ownership already is. +surface keeps the sparse sealed-transition apply where the fleet ownership +already is. The supervisor crate currently deps **only** `lance-graph-contract` (NOT planner). The driver requires the planner's `persist_sink`, `owner_adapter`, and @@ -234,7 +276,7 @@ slice** the `persist_sink` probes use. This is honest and deliberate: > durability leg still fake" is the accurate status — `compile+test green ≠ > storage proven` (the Ladybug rule). The P4a…e falsifiers all pass against > the fake sink because they probe the *control* invariants (one seal, one -> version, inline fan, round-trip, watermark idempotence), none of which need +> version, inline sparse apply, round-trip, watermark idempotence), none of which need > real crash durability. Only P4f-real-durability would need the concrete > sink, and P4f as specified is a fan-out **scale** measurement, not a > durability claim. @@ -254,9 +296,12 @@ shader here. ## 6. 64k mechanics + the W2a scale gate **Scale framing** (per `mailbox-cycle-aware-write-contract-v1.md`): one basin = -one prefix table = **16k mailboxes**; **64k = ~4 basins** = the sweep target. -The fan-step (§3) is a straight pass over the fleet SoA, so 64k mailboxes = one -sync sweep of ~4 prefix tables, not 64k async operations. +one prefix table = **16k mailboxes**; **64k = ~4 basins** = the registered-fleet +size. **The apply cost scales with the SPARSE sealed-transition set, not the +fleet.** The supervisor iterates only the cycle's sealed `paired_move` SweepSlots +(§3.1) — 17 dirty owners cost 17 `try_advance_phase` calls, not a 64k sweep — and +the other ~64k owners are never touched. This is the whole point of the +sparse-cycle ruling: registration is 64k; mutation is the dirty subset. **Why `persist_sink`'s guarantees make 64k concurrent casts safe:** @@ -272,10 +317,11 @@ sync sweep of ~4 prefix tables, not 64k async operations. **W2a scale gate (D-V3-W2a, board-as-tenant, currently GATED/deferred):** the driver targets the **existing `MailboxSoaView::phase()` surface today** and adopts the per-mailbox board **tenant column** (kanban board as `ValueTenant`) -when W2a un-gates. W2a is a **scale / cleanliness gate** — the fan-out becomes a -tenant *column read* instead of per-mailbox structs — **NOT a hard blocker** for -the control-loop shape. The loop closes on the `phase()` surface now; W2a makes -the 64k fan cheaper and cleaner later. This is exactly why **P4f is gated on +when W2a un-gates. W2a is a **scale / cleanliness gate** — resolving the sealed +owners becomes a tenant *column read* instead of per-mailbox structs — **NOT a +hard blocker** for the control-loop shape. The loop closes on the `phase()` +surface now; W2a makes owner-resolution over the fleet cheaper and cleaner later. +This is exactly why **P4f is gated on W2a** and labelled a scale gate, while P4a…e are not. --- @@ -314,9 +360,138 @@ W2a** and labelled a scale gate, while P4a…e are not. | **CycleDriver** (P4a…f — closes seal→step→think→cast) | **PLANNED / CONJECTURE** — this plan; probe-gated | | Home = `lance-graph-supervisor` + new planner path-dep (fallback: planner) | **DECIDED** (§5.1) — verify no cycle via `cargo tree` | | Durability leg (concrete `LanceShardSink`, real crash durability) | **DEFERRED** — driver wires the contract-probe fake; control loop closes regardless | -| Board-as-tenant fan-out (D-V3-W2a) | **GATED** — driver uses `phase()` today; P4f scale gate adopts the tenant column when W2a un-gates | +| Board-as-tenant owner-resolution (D-V3-W2a) | **GATED** — driver uses `phase()` today; P4f scale gate adopts the tenant column when W2a un-gates | The organs exist; the loop does not. This plan is the record of the one seam that makes the merged persistence bootstrap load-bearing — and of the honest boundary that the control loop closes now while the durability leg stays a fake until the crash falsifiers earn the concrete sink. + +--- + +## 9. D-MBX crate-responsibility map (the production spine — ratified) + +The canonical ownership map. Verified against `Cargo.toml` deps 2026-08-02 (see +§5.2). The distinction that must stay explicit: **MailboxSoA type/layout home = +`cognitive-shader-driver`; exclusive runtime ownership = `lance-graph-supervisor`; +decision + persistence-contract home = `lance-graph-planner`.** + +| Crate | Owns | Explicitly does NOT own | +|---|---|---| +| **lance-graph-contract** | Canonical shared types: `KanbanColumn` / `KanbanMove`, `DatasetVersion`, the `VersionScheduler` traits, `MailboxSoaView` / `MailboxSoaOwner`, legal Rubicon transitions. Zero-dep. | fleet ownership; persistence implementation; any thought body. | +| **cognitive-shader-driver** | The canonical **MailboxSoA layout**; native cognition / shader / thinking machinery; thinking atoms, thinking styles, SoA columns — **defines the anatomy**. | the production fleet **lifecycle** (it does not run the runtime loop; its optional `lance-graph-planner` dep is debug/serve DTOs, not fleet ownership). | +| **lance-graph-planner** | **Decides what should happen**: StyleStrategy + StrategyOutcome, the intended `KanbanMove`, `owner_adapter`, `BatchWriter` cast/intents, cycle collection + coalescing, `persist_cycle` / `WalSink` contract, recovery + temporal projection **contracts**. | **never directly mutates a supervisor-owned MailboxSoA**; never depends on supervisor. | +| **lance-graph-supervisor** | **Production runtime owner of MailboxSoA instances**: `KanbanActor` state IS the owner; authoritative phase mutation; the production cycle-loop composition (**D-MBX-A6-P4**); applies only the **sealed sparse transitions**; fires CognitiveWork; returns Outcomes to planner/`owner_adapter`. | it consumes planner; planner never consumes it. | +| **lance-graph** | The actual Lance dataset + `DatasetVersion` substrate; the external-reader version subscription (`LanceVersionScheduler`); the future concrete `LanceShardSink` / physical persistence. | it is a **storage substrate, not a cognitive fleet owner**. | + +**D-id allocation (audited):** D-MBX-A1..A5 → `cognitive-shader-driver` (+contract +support); D-MBX-A6-P1/P2 → `lance-graph-contract`; D-MBX-A6-P3a..P3e → +`lance-graph-planner`; **D-MBX-A6-P4a..P4f → `lance-graph-supervisor`** (one-way +dep on planner); D-MBX-9-IN contract → `lance-graph-contract`; D-MBX-9-IN +external-reader impl → `lance-graph`; D-V3-W2b `KanbanActor` → +`lance-graph-supervisor`. + +**Dependency direction (verified 2026-08-02, §5.2):** +`lance-graph-supervisor → lance-graph-planner → lance-graph-contract`; +`lance-graph-planner → lance-graph-contract`. **Planner must not depend on +supervisor.** Currently supervisor deps only contract (+ callcenter); the +`supervisor → planner` edge is the **planned P4 wiring** and is acyclic (planner's +dep closure never reaches supervisor). `cognitive-shader-driver` has an optional +(feature-gated) planner dep for debug/serve DTOs — not fleet ownership, not a +cycle. + +--- + +## 10. Adjacent-crates doctrine — bystanders, basements, and adapters (NOT owners) + +The production spine (§9) is a straight railway track: +**contract defines · planner proposes and seals · supervisor owns and applies · +shader thinks · Lance persists.** Adjacent crates **observe, adapt, or provide +optional capabilities** — none is crowned emperor of the hippocampus. + +### 10.1 `symbiont` — golden-image + bystander research laboratory + +**Allowed:** full-stack compile/link golden image; integration + scale probes; +brainstorming + falsification playground; possible AST-arm experiments +(Elixir-shaped syntax without an Elixir runtime, SurrealQL DDL/expression AST, +OGAR adapter composition); a possible second research leg for +`lance-graph-arm-discovery`, grammar heuristics, time-series observation, +cross-system hypothesis generation. + +**Forbidden:** authoritative MailboxSoA owner; production D-MBX scheduler; +production Kanban lifecycle; production WAL owner; independent version authority; +second source of truth for cognition; a **required dependency** of planner, +supervisor, or cognitive-shader-driver. + +`SymbiontBoard` (D2) impls `MailboxSoaView`/`MailboxSoaOwner` and its +`step()`-advances-every-board loop is **probe-only and intentionally local** — a +shape-prover, never the production owner. Any reusable production logic found in +symbiont is classified as *probe-only-and-local* or *candidate for later +extraction into its canonical D-MBX crate* — **not extracted in this task**. +Symbiont may observe the brain, suggest patterns to it, and test combinations +around it; it must not become the alien twin driving the hands. + +### 10.2 `rs-graph-llm` — optional capability basement + +**Useful:** agentic-coding-shaped demonstrations; sparse LLM assistance; ticket +orchestration; Rig integration; OpenClaw / tool-use adapters; optional +CognitiveWork capability providers; human-in-the-loop workflow façades. + +**Forbidden:** authoritative MailboxSoA storage; authoritative Kanban state; a +second planning lifecycle; a second WAL or version ledger; mirrored live +cognition state; a **required dependency** of D-MBX core crates. + +Composition shape: `application / MedCare composition layer` sits **above** both +the D-MBX production runtime and the optional `rs-graph-llm` / Rig capabilities — +NOT `lance-graph core → rs-graph-llm → duplicated session/Kanban/storage state`. +When `rs-graph-llm` invokes D-MBX thinking it is a **client / capability +provider**; when D-MBX invokes an LLM/tool capability the result returns as an +**Outcome or evidence input**. `rs-graph-llm` **never owns the standing wave** +(consistent with the workspace rule that rig is membrane-tier, not a brain crate). + +### 10.3 `ogar-*` universal adapters — AST / declaration / adapter basement + +**Allowed:** source AST ingestion; Elixir / Ruby / Python / SQL / SurrealQL +adaptation; Class / ActionDef declaration surfaces; code generation; cold-path +capability descriptions; schema + behaviour translation. + +**Forbidden:** live MailboxSoA ownership; an independent D-MBX scheduler; +duplicate runtime Kanban state; standing-wave persistence. + +OGAR may **describe available behaviour**; it does not own the living cognitive +cycle. + +--- + +## 11. Subagent anti-drift guardrail (paste into every D-MBX worker brief) + +Before any subagent changes D-MBX code, it MUST answer: + +1. Is this **shared vocabulary, planning, runtime ownership, cognition, or + storage**? +2. Which **canonical crate** (§9) owns that responsibility? +3. Does the change create a **second** owner / scheduler / WAL / Kanban lifecycle + / Session state / `DatasetVersion` authority / MailboxSoA representation? +4. Is **symbiont** being used as a production dependency merely because it already + links many crates? +5. Is **rs-graph-llm** being allowed to mirror or own live SoA state merely + because it already has workflow/session abstractions? +6. Could the change be an **adapter, callback, Outcome, or trait seam** instead of + importing a whole neighbouring runtime? + +**STOP and report (do not proceed) when:** + +- planner would depend on supervisor; +- a D-MBX **core** crate would depend on symbiont or rs-graph-llm; +- a **second** Kanban phase field appears; +- a **second** cycle/version counter appears; +- a Session snapshot becomes authoritative over MailboxSoA; +- **a `DatasetVersion` tick advances every owner** (the sparse-cycle violation + this revision fixed — §3.1); +- SurrealDB JSON becomes the live cognition representation; +- a **production type is first declared inside symbiont**. + +The desired result is a straight railway track, not a grand unification: +**contract defines · planner proposes and seals · supervisor owns and applies · +shader thinks · Lance persists**; adjacent crates observe, adapt, or provide +optional capabilities. From c82a336743aee5efc08b83ce03c367208d317ac4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 10:52:06 +0000 Subject: [PATCH 3/6] =?UTF-8?q?feat(supervisor):=20D-MBX-A6-P4a+P4b=20?= =?UTF-8?q?=E2=80=94=20cycle=20loop-closure=20driver=20(persist=5Fsink's?= =?UTF-8?q?=20first=20production=20caller)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the ZERO-caller gap on the merged #878 persist_sink: nothing drove the seal→apply loop, so the sealed version never advanced any owner. New module `lance-graph-supervisor::cycle_driver`, behind the `cycle-driver` feature (optional ONE-WAY `lance-graph-planner` path-dep — planner never deps supervisor, verified acyclic; default supervisor build stays light, no planner/ractor). Mints NO domain types — composes the shipped organs: - P4a: `collect_casts(writer, cycle, row_of)` drains a `BatchWriter>`'s staged casts into `Vec` (one slot/cast; stream_position = CastId; paired_move = first intended move). `seal_cycle(sink, frame, casts)` reads out the SPARSE `SealedTransition` set (only slots with a move, stream-ordered) then `persist_cycle` -> exactly one WAL write, one DatasetVersion. - P4b: `apply_sealed_transitions(fleet, &SealedCycle)` iterates ONLY the sealed sparse set, resolves each owner via the `MailboxFleet` trait (blanket-impl'd for HashMap), applies one legal `try_advance_phase`; EVERY unrepresented owner stays byte-identical (never resolved). Interim rule: <=1 durable transition per owner per cycle (2nd same-owner move -> `deferred`); unknown owner -> `missing` (counted, not a crash); StalePhase/OwnerMismatch guards; reads NO dataset (version already sealed — no scan_sealed/versions/ drive_once). `run_cycle` = P4a->P4b convenience. The load-bearing rule, enforced in code: a DatasetVersion is GLOBAL KNOWLEDGE, NOT permission to advance every mailbox — only the sealed sparse set advances (E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1 + E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1). Tests: 7 cycle_driver lib tests (10 total in the crate), headline = the 64k/17 falsifier — 65_536 mailboxes, 17 sealed transitions -> exactly 17 advance, the other 65_519 byte-identical, one WAL write, ZERO dataset reads. Plus one-WAL-write amortization, empty-sparse-set-advances-nobody, interim-defer, StalePhase corruption, missing-owner-counted, run_cycle round-trip. clippy (--features cycle-driver) exit 0, fmt clean; default (no-feature) build unchanged. Durability leg stays the contract-probe fake (FakeWalSink) — control loop closed, storage NOT proven (Ladybug rule); concrete LanceShardSink still deferred. Cargo.lock updated for the new optional supervisor->planner dep. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; STATUS_BOARD D-MBX-A6-P4 -> P4a+P4b Shipped (slice); LATEST_STATE prepended. Remaining: P4c (CognitiveWork thought body + cast round-trip), P4d/P4e/P4f. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 8 + .claude/board/STATUS_BOARD.md | 2 +- Cargo.lock | 15 +- crates/lance-graph-supervisor/Cargo.toml | 7 + .../src/cycle_driver.rs | 658 ++++++++++++++++++ crates/lance-graph-supervisor/src/lib.rs | 12 + 6 files changed, 699 insertions(+), 3 deletions(-) create mode 100644 crates/lance-graph-supervisor/src/cycle_driver.rs diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index d8cfcc1a..9ec30f2a 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,11 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P4a+P4b: cycle loop-closure driver (persist_sink's first production caller) + +- `lance-graph-supervisor::cycle_driver` (NEW module, behind feature **`cycle-driver`** = optional one-way `lance-graph-planner` dep; default supervisor build stays light, no planner/ractor) — **the seam that makes the merged #878 `persist_sink` load-bearing** (it was the ZERO-caller gap). Supervisor is the runtime fleet owner; planner decides; the dep is one-way (planner never deps supervisor — verified acyclic). Mints **NO** domain types — composes `SweepSlot`/`CycleFrame`/`CycleId`/`persist_cycle`/`WalSink`/`PersistError` (planner) + `BatchWriter` (planner) + `MailboxSoaOwner`/`KanbanMove`/`DatasetVersion`/`MailboxId` (contract). + - **P4a** — `collect_casts(writer, cycle, row_of) -> Vec` drains a `BatchWriter>`'s staged casts (one slot/cast; `stream_position`=`CastId`; `paired_move`=first intended move); `seal_cycle(sink, frame, casts) -> SealedCycle{version, transitions}` reads out the **sparse** `SealedTransition` set (only slots carrying a move, stream-ordered) then `persist_cycle` → **exactly one WAL write, one `DatasetVersion`**. + - **P4b** — `apply_sealed_transitions(fleet, &SealedCycle) -> AppliedCycle{version, applied, deferred, missing}`: iterate **ONLY the sealed sparse transitions**, resolve each owner via the `MailboxFleet` trait (blanket-impl'd for `HashMap`), apply **one** legal `try_advance_phase`; **every unrepresented owner is byte-identical** (never resolved). Interim **≤1 durable transition/owner/cycle** (second same-owner move → `deferred`); unknown owner → `missing` (counted, not a crash); `StalePhase`/`OwnerMismatch` corruption guards; **reads NO dataset** (version already sealed by P4a — no `scan_sealed`/`versions`/`drive_once`). `run_cycle(...)` = P4a→P4b convenience. +- **The load-bearing rule enforced in code:** a `DatasetVersion` is global knowledge, NOT permission to advance every mailbox — only the sealed sparse set advances (`E-D-MBX-SPINE-...-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-...-1`). +- **7 lib tests green** (`cargo test -p lance-graph-supervisor --features cycle-driver`), headline = **64k/17 falsifier**: 65 536 mailboxes, 17 sealed transitions → exactly 17 advance (→CognitiveWork, cycle bumped), the other 65 519 byte-identical, one WAL write, **zero dataset reads**. Plus: one-WAL-write amortization, empty-sparse-set advances nobody, interim-defer, StalePhase corruption, missing-owner-counted, run_cycle round-trip. clippy (feature) exit 0, fmt clean; default build unchanged. **Durability leg still the contract-probe fake** (`FakeWalSink`) — control loop closed, storage NOT proven (Ladybug rule); concrete `LanceShardSink` still deferred. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; STATUS_BOARD D-MBX-A6-P4 → P4a+P4b Shipped(slice). Remaining: P4c (CognitiveWork thought body + cast round-trip through owner_adapter), P4d/P4e/P4f. + ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P3e persistence-sink reshaped to WAL-amortized cycle (one write per sweep) - `lance_graph_planner::persist_sink` — **reshaped in place** (PR #878) from the per-cast durable-witness model to the **cycle/WAL** model (operator ruling on the seam-shape fork surfaced 2026-08-01). The durable unit is the **cycle/sweep**, NOT the cast: 64k concurrent thoughts stage into an owned `Vec`, `persist_cycle(sink, frame, casts)` folds+freezes them, and `WalSink::commit_cycle(base, DetachedCycleBatch)` does **exactly one atomic append → one `DatasetVersion`** (WAL amortization — 64k thoughts, one write). Types: `CycleId`, `CycleFrame{cycle, base_version}` (storage-only — NO rung/branch/semantic tags), `SweepSlot{cycle, stream_position, owner, row, paired_move, payload}` (boring landing, no `basis`), `LandedSlot{version, slot}`, `DetachedCycleBatch{frame, landings, image}` (frozen, deinterlaced, coalesced). **Write-side ordering** = `order_cycle_stably(rows, key)` (generic over the caller's canonical key; stable-orders casts by the EXISTING `stream_position` in `freeze` BEFORE the append) — completion order never becomes storage order (physical race); `scan_sealed` returns stored order and NEVER sorts. **Epistemic horizon** = sealed read (`read Vn / write Vn+1`); an uncommitted cycle is invisible to `scan_sealed`. **Coalescing** = real per-row fold (`row -> last payload in stream order`), not last-chunk-wins. `recover_and_apply(owner, sealed, applied_through)` + the durable **watermark** idempotence + `StalePhase`/`OwnerMismatch` guards survive unchanged. `versions()` = the cheap coarse timeline a downstream time-series consumer (stockfish-rs, another session) looks up, no landing replay. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 53033e01..f940a092 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -964,7 +964,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | | D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **Merged** | #878 (merged; reshaped in place to the cycle/WAL model = P3e — `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`; + the §2 sparse-delta storage ruling `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`, RATIFIED/UNIMPLEMENTED); ordering/recovery CONTRACT probed (348 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | -| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **Queued (PLANNED)** | plan `.claude/plans/cycle-loop-closure-driver-v1.md`; home lance-graph-supervisor (new one-way planner dep, verified acyclic; planner fallback); CONTROL loop closes, durability leg stays the contract-probe fake until concrete `LanceShardSink`; consumes #878 persist_sink; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` | +| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a+P4b Shipped (slice); P4c–f Queued** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** = `collect_casts` (drain `BatchWriter` → `Vec`) + `seal_cycle` (`persist_cycle` → one WAL write / one `DatasetVersion` + the sparse `SealedTransition` set). **P4b** = `apply_sealed_transitions` (iterate ONLY the sealed sparse set; each represented owner one legal `try_advance_phase`; unrepresented owners byte-identical; interim ≤1/owner/cycle → `deferred`; unknown owner → `missing`; reads NO dataset). `run_cycle` convenience. 7 lib tests incl. the **64k/17 falsifier** (65 536 mailboxes, 17 sealed → exactly 17 advance, 65 519 byte-identical, one WAL write, zero dataset reads); clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`MailboxSoaOwner`). Durability leg still the contract-probe fake. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/Cargo.lock b/Cargo.lock index 29dd59fa..bab06d39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4825,6 +4825,7 @@ dependencies = [ "lance", "lance-arrow", "lance-graph-catalog", + "lance-graph-cognitive", "lance-graph-contract 0.1.0", "lance-graph-planner", "lance-index", @@ -4909,6 +4910,15 @@ dependencies = [ "wiremock", ] +[[package]] +name = "lance-graph-cognitive" +version = "0.1.0" +dependencies = [ + "holograph", + "lance-graph-contract 0.1.0", + "ndarray 0.17.2", +] + [[package]] name = "lance-graph-consumer-conformance" version = "0.1.0" @@ -4933,7 +4943,7 @@ dependencies = [ [[package]] name = "lance-graph-contract" version = "0.1.0" -source = "git+https://github.com/AdaWorldAPI/lance-graph?branch=claude%2Fogar-docir-architecture-jjzlig#241c7eff167264bafd4f458858e5495dd2afdd9b" +source = "git+https://github.com/AdaWorldAPI/lance-graph?branch=main#18b5b65414ede0682445693d6d08449da64f1ce1" dependencies = [ "glob", "serde", @@ -5007,6 +5017,7 @@ dependencies = [ "cognitive-shader-driver", "lance-graph-callcenter", "lance-graph-contract 0.1.0", + "lance-graph-planner", "ractor", "static_assertions", "thiserror 1.0.69", @@ -6052,7 +6063,7 @@ dependencies = [ name = "ogar-class-view" version = "0.1.0" dependencies = [ - "lance-graph-contract 0.1.0 (git+https://github.com/AdaWorldAPI/lance-graph?branch=claude%2Fogar-docir-architecture-jjzlig)", + "lance-graph-contract 0.1.0 (git+https://github.com/AdaWorldAPI/lance-graph?branch=main)", "ogar-vocab", ] diff --git a/crates/lance-graph-supervisor/Cargo.toml b/crates/lance-graph-supervisor/Cargo.toml index 012a053d..e32b1a38 100644 --- a/crates/lance-graph-supervisor/Cargo.toml +++ b/crates/lance-graph-supervisor/Cargo.toml @@ -12,6 +12,9 @@ description = "ractor-supervised actor tree for the callcenter membrane (PR-G2, [dependencies] lance-graph-callcenter = { path = "../lance-graph-callcenter" } lance-graph-contract = { path = "../lance-graph-contract" } +# P4 loop-closure driver (feature `cycle-driver`): the ONE-WAY dep on the planner +# (persist_sink + batch_writer). Planner does NOT dep supervisor — no cycle. +lance-graph-planner = { path = "../lance-graph-planner", optional = true } thiserror = "1" tracing = "0.1" @@ -41,6 +44,10 @@ tokio = { version = "1", features = ["rt", "time", "macros"], option default = [] supervisor = ["dep:ractor", "dep:static_assertions", "dep:tokio"] supervisor-lifecycle-audit = ["supervisor"] +# P4a/P4b cycle loop-closure driver — pulls the planner (persist_sink + batch_writer). +# Independent of the ractor `supervisor` feature; needs tokio only for the async +# persist_cycle I/O leg in tests. +cycle-driver = ["dep:lance-graph-planner", "dep:tokio"] [dev-dependencies] tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/crates/lance-graph-supervisor/src/cycle_driver.rs b/crates/lance-graph-supervisor/src/cycle_driver.rs new file mode 100644 index 00000000..23fb2852 --- /dev/null +++ b/crates/lance-graph-supervisor/src/cycle_driver.rs @@ -0,0 +1,658 @@ +//! P4a / P4b — the cycle loop-closure driver (supervisor-side). +//! +//! This is the seam that makes the merged `persist_sink` cycle/WAL bootstrap +//! **load-bearing**. Before this, `persist_sink::persist_cycle` had zero +//! production callers. The driver composes the already-shipped organs — it mints +//! **no** new semantic / temporal / rung / witness type: +//! +//! - **P4a** ([`collect_casts`] + [`seal_cycle`]): drain the fleet's staged +//! planner casts into one `Vec`, `persist_cycle` them → **exactly +//! one WAL write → one `DatasetVersion`**, and read out the **sparse** sealed +//! paired-transition set (only the owners that actually cast a move). +//! - **P4b** ([`apply_sealed_transitions`]): iterate **only** the sealed sparse +//! transitions, resolve each owner in the fleet, apply **one** legal +//! `try_advance_phase`. **Every unrepresented owner is left byte-identical.** +//! +//! ## The load-bearing rule (E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1) +//! +//! A `DatasetVersion` is **global knowledge**, NOT permission to advance every +//! mailbox. Kanban mutation is **sparse and owner-specific**: an owner advances +//! only because it produced a sealed `paired_move`. The version tick never fans a +//! step across the fleet. Applying only the sealed sparse set is what keeps a +//! globally-complete cycle physically sparse +//! (`E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). +//! +//! **Interim conservative rule:** at most **one** durable phase transition per +//! owner per sealed cycle. A second sealed transition for an already-advanced +//! owner is **deferred** (held for the next sealed horizon), not applied. +//! +//! ## Ownership (D-MBX spine) +//! +//! The supervisor is the **exclusive runtime owner** of the fleet; the planner +//! **decides** (produces the casts + the persistence contract). This driver lives +//! in the supervisor and depends **one-way** on the planner (planner never deps +//! supervisor — no cycle). It applies through `MailboxSoaOwner::try_advance_phase` +//! directly — no ractor message bus needed for P4a/P4b. +//! +//! ## Honesty +//! +//! The control loop closes here; the **durability leg stays the contract-probe +//! fake** until the concrete `LanceShardSink` lands (`compile+test green ≠ +//! storage proven`, the Ladybug rule). `apply_sealed_transitions` reads **no** +//! dataset — the version was already sealed by `seal_cycle`. + +use std::collections::{HashMap, HashSet}; + +use lance_graph_contract::collapse_gate::MailboxId; +use lance_graph_contract::kanban::KanbanMove; +use lance_graph_contract::scheduler::DatasetVersion; +use lance_graph_contract::soa_view::{MailboxSoaOwner, MailboxSoaView}; + +use lance_graph_planner::batch_writer::BatchWriter; +use lance_graph_planner::persist_sink::{ + persist_cycle, CycleFrame, CycleId, PersistError, SweepSlot, WalSink, +}; + +/// One sealed paired transition — a member of the **sparse** set a sealed cycle +/// publishes. Carries `stream_position` so P4b applies transitions in canonical +/// order (later-position wins the interim one-per-owner slot). Boring by design: +/// it says *which owner advances where*, nothing else. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SealedTransition { + /// The canonical order key inherited from the cast's `SweepSlot`. + pub stream_position: u64, + /// The owner that produced this transition (== `mv.mailbox`). + pub owner: MailboxId, + /// The lifecycle move the owner cast. + pub mv: KanbanMove, +} + +/// P4a output — one sealed cycle: the single published `DatasetVersion` and the +/// **sparse** sealed paired-transition set. `transitions` is empty for a no-op +/// cycle (no owner cast a move); its length is ≤ the number of participants and +/// is typically ≪ the fleet size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SealedCycle { + /// The one version this cycle sealed into. + pub version: DatasetVersion, + /// Only the owners that cast a `paired_move` — the sparse subset. + pub transitions: Vec, +} + +/// P4b output — the effect of applying a sealed cycle's sparse transition set. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppliedCycle { + /// The version whose sealed transitions were applied. + pub version: DatasetVersion, + /// One move per **advanced** owner (distinct owners; interim ≤1 per cycle). + pub applied: Vec, + /// Extra same-owner sealed transitions held for the next horizon (interim + /// one-transition-per-owner rule). + pub deferred: usize, + /// Sealed transitions whose owner is not registered in the fleet (a + /// registration gap — surfaced as a count, not a crash; the durable move + /// stays in the log for a later recovery pass). + pub missing: usize, +} + +/// A resolvable collection of mailbox owners — the supervisor's production fleet. +/// P4b resolves each sealed transition's owner through this; **unrepresented +/// owners are never resolved and never touched** (that is what keeps them +/// byte-identical). +pub trait MailboxFleet { + /// The concrete owner type this fleet holds. + type Owner: MailboxSoaOwner; + /// Resolve a mailbox to its owner, or `None` if it is not registered. + fn owner_mut(&mut self, id: MailboxId) -> Option<&mut Self::Owner>; +} + +/// A `HashMap` keyed by `MailboxId` is the simplest concrete fleet — used by the +/// production supervisor's owner registry and by the tests. +impl MailboxFleet for HashMap { + type Owner = O; + fn owner_mut(&mut self, id: MailboxId) -> Option<&mut O> { + self.get_mut(&id) + } +} + +/// **P4a (drain).** Map the fleet's staged `BatchWriter` casts into `SweepSlot`s +/// for one cycle. One slot per cast: `stream_position` = the monotonic `CastId` +/// (cast order), `paired_move` = the cast's **first** intended move (interim: one +/// intended move per cast), `row` resolved by the caller's `row_of`, `payload` = +/// the drained descriptor bytes. Draining clears the writer's payload staging. +/// +/// Casts with no recorded move become no-step landings (`paired_move = None`) — +/// they still coalesce into the cycle image but contribute no transition. +#[must_use] +pub fn collect_casts( + writer: &mut BatchWriter>, + cycle: CycleId, + mut row_of: impl FnMut(MailboxId) -> u64, +) -> Vec { + // Drain first (ends the &mut borrow), then read the intent moves immutably. + let drained: Vec<_> = writer.drain_pending_payloads().collect(); + drained + .into_iter() + .filter_map(|(cast, payload)| { + let owner = writer.on_behalf_of(cast)?; + let paired_move = writer + .intent_moves(cast) + .and_then(|moves| moves.first().copied()); + Some(SweepSlot { + cycle, + stream_position: cast.0, + owner, + row: row_of(owner), + paired_move, + payload, + }) + }) + .collect() +} + +/// **P4a (seal).** Freeze the collected casts into one cycle: read out the +/// **sparse** sealed paired-transition set (the slots that carry a `paired_move`, +/// ordered by `stream_position`), then `persist_cycle` → **exactly one WAL write, +/// one `DatasetVersion`**. `persist_cycle`'s own guards still run +/// (cross-owner-move reject, cast/frame cycle-match). The version is published +/// before any owner advances (P4b). +pub async fn seal_cycle( + sink: &S, + frame: CycleFrame, + casts: Vec, +) -> Result { + // Read the transitions out BEFORE `persist_cycle` takes ownership of `casts`. + let mut transitions: Vec = casts + .iter() + .filter_map(|s| { + s.paired_move.map(|mv| SealedTransition { + stream_position: s.stream_position, + owner: s.owner, + mv, + }) + }) + .collect(); + transitions.sort_by_key(|t| t.stream_position); + let version = persist_cycle(sink, frame, casts).await?; + Ok(SealedCycle { + version, + transitions, + }) +} + +/// **P4b.** Apply **only** the sealed sparse transition set to the fleet. Iterate +/// the transitions in canonical `stream_position` order; resolve each owner and +/// apply **one** legal `try_advance_phase`. **Every unrepresented owner is left +/// byte-identical** — it is never resolved, never touched. +/// +/// Interim rule: at most one durable transition per owner per cycle — a second +/// sealed transition for an already-advanced owner is **deferred**. A sealed +/// move whose `from` no longer matches the owner's phase is corruption +/// ([`PersistError::StalePhase`]); a cross-owner move is +/// [`PersistError::OwnerMismatch`] (defence in depth — `persist_cycle` already +/// rejected it). On such an error the fleet is left mid-apply; re-drive from the +/// persisted watermark via `persist_sink::recover_and_apply`. +/// +/// Reads **no** dataset: the version was sealed by [`seal_cycle`]; this is a pure +/// in-memory fan over the sparse set (no `scan_sealed`, no `versions`, no +/// `drive_once`). +pub fn apply_sealed_transitions( + fleet: &mut F, + sealed: &SealedCycle, +) -> Result { + let mut ordered: Vec<&SealedTransition> = sealed.transitions.iter().collect(); + ordered.sort_by_key(|t| t.stream_position); + + let mut applied = Vec::new(); + let mut advanced: HashSet = HashSet::new(); + let mut deferred = 0usize; + let mut missing = 0usize; + + for t in ordered { + // Interim ≤1-per-owner: a later sealed move for an already-advanced owner + // waits for the next sealed horizon. + if advanced.contains(&t.owner) { + deferred += 1; + continue; + } + let Some(owner) = fleet.owner_mut(t.owner) else { + missing += 1; + continue; + }; + // Defence in depth (persist_cycle already validated at seal time). + if t.mv.mailbox != t.owner { + return Err(PersistError::OwnerMismatch { + move_owner: t.mv.mailbox, + landing_owner: t.owner, + }); + } + // Corruption guard: the sealed move's `from` must match the owner's phase. + if t.mv.from != owner.phase() { + return Err(PersistError::StalePhase { + owner_phase: owner.phase(), + move_from: t.mv.from, + }); + } + let step = owner + .try_advance_phase(t.mv.to) + .map_err(PersistError::Illegal)?; + applied.push(step); + advanced.insert(t.owner); + } + + Ok(AppliedCycle { + version: sealed.version, + applied, + deferred, + missing, + }) +} + +/// Convenience: run one full cycle — **P4a** (drain the writer + seal) then +/// **P4b** (apply the sparse set). The one seam a running loop calls per cycle. +/// Returns both the sealed cycle and the applied effect. +pub async fn run_cycle( + sink: &S, + fleet: &mut F, + writer: &mut BatchWriter>, + frame: CycleFrame, + row_of: impl FnMut(MailboxId) -> u64, +) -> Result<(SealedCycle, AppliedCycle), PersistError> +where + S: WalSink, + F: MailboxFleet, +{ + let casts = collect_casts(writer, frame.cycle, row_of); + let sealed = seal_cycle(sink, frame, casts).await?; + let applied = apply_sealed_transitions(fleet, &sealed)?; + Ok((sealed, applied)) +} + +#[cfg(test)] +mod tests { + use super::*; + use lance_graph_contract::kanban::{ExecTarget, KanbanColumn}; + use lance_graph_contract::soa_view::MailboxSoaView; + use lance_graph_planner::persist_sink::{DetachedCycleBatch, LandedSlot, WriteFailed}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::Mutex; + + // ── Minimal in-RAM owner (mirrors the persist_sink FakeOwner) ─────────────── + #[derive(Clone, PartialEq, Eq, Debug)] + struct FakeOwner { + id: MailboxId, + phase: KanbanColumn, + cycle: u32, + } + impl FakeOwner { + fn at(id: MailboxId, phase: KanbanColumn) -> Self { + Self { + id, + phase, + cycle: 0, + } + } + } + impl MailboxSoaView for FakeOwner { + fn mailbox_id(&self) -> MailboxId { + self.id + } + fn n_rows(&self) -> usize { + 0 + } + fn w_slot(&self) -> u8 { + 0 + } + fn current_cycle(&self) -> u32 { + self.cycle + } + fn phase(&self) -> KanbanColumn { + self.phase + } + fn energy(&self) -> &[f32] { + &[] + } + fn edges_raw(&self) -> &[u64] { + &[] + } + fn meta_raw(&self) -> &[u32] { + &[] + } + fn entity_type(&self) -> &[u16] { + &[] + } + } + impl MailboxSoaOwner for FakeOwner { + fn advance_phase(&mut self, to: KanbanColumn) -> KanbanMove { + let from = self.phase; + self.phase = to; + self.cycle = self.cycle.wrapping_add(1); + KanbanMove { + mailbox: self.id, + from, + to, + witness_chain_position: self.cycle, + exec: ExecTarget::Native, + } + } + } + + // ── The WAL sink fake — counts writes AND reads (base-fenced) ──────────────── + struct SealedRec { + frame: CycleFrame, + version: DatasetVersion, + } + struct FakeWalSink { + sealed: Mutex>, + next_version: AtomicU64, + wal_writes: AtomicU64, + reads: AtomicU64, // scan_sealed + versions — MUST stay 0 across P4a+P4b + } + impl FakeWalSink { + fn new() -> Self { + Self { + sealed: Mutex::new(Vec::new()), + next_version: AtomicU64::new(1), + wal_writes: AtomicU64::new(0), + reads: AtomicU64::new(0), + } + } + fn wal_writes(&self) -> u64 { + self.wal_writes.load(Ordering::SeqCst) + } + fn reads(&self) -> u64 { + self.reads.load(Ordering::SeqCst) + } + } + impl WalSink for FakeWalSink { + async fn commit_cycle( + &self, + base: DatasetVersion, + batch: DetachedCycleBatch, + ) -> Result { + let mut sealed = self.sealed.lock().unwrap(); + let head = sealed.last().map_or(DatasetVersion(0), |s| s.version); + if base != head { + return Err(WriteFailed(format!("stale base {base:?}, head {head:?}"))); + } + self.wal_writes.fetch_add(1, Ordering::SeqCst); + let version = DatasetVersion(self.next_version.fetch_add(1, Ordering::SeqCst)); + sealed.push(SealedRec { + frame: batch.frame, + version, + }); + Ok(version) + } + async fn scan_sealed( + &self, + _from: Option, + ) -> Result, WriteFailed> { + self.reads.fetch_add(1, Ordering::SeqCst); + Ok(Vec::new()) + } + async fn versions(&self) -> Result, WriteFailed> { + self.reads.fetch_add(1, Ordering::SeqCst); + Ok(self + .sealed + .lock() + .unwrap() + .iter() + .map(|s| (s.frame.cycle, s.version)) + .collect()) + } + } + + fn mv(owner: MailboxId, from: KanbanColumn, to: KanbanColumn) -> KanbanMove { + KanbanMove { + mailbox: owner, + from, + to, + witness_chain_position: 0, + exec: ExecTarget::Native, + } + } + + /// Stage one Planning→CognitiveWork cast for each of `owners`. + fn writer_with_moves(owners: &[MailboxId]) -> BatchWriter> { + let mut w: BatchWriter> = BatchWriter::new(); + for &o in owners { + w.cast( + o, + vec![mv(o, KanbanColumn::Planning, KanbanColumn::CognitiveWork)], + vec![0xAB], + ); + } + w + } + + // ── P4a FALSIFIER: drain N casts → exactly one WAL write, one version ─────── + #[tokio::test] + async fn p4a_drains_casts_and_seals_one_wal_write_one_version() { + let sink = FakeWalSink::new(); + let owners: Vec = (0..100).collect(); + let mut w = writer_with_moves(&owners); + let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); + assert_eq!(casts.len(), 100, "one slot per staged cast"); + assert_eq!(sink.wal_writes(), 0, "collecting writes no WAL"); + + let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + assert_eq!(sink.wal_writes(), 1, "100 casts → exactly ONE WAL write"); + assert_eq!(sealed.version, DatasetVersion(1), "→ exactly one version"); + assert_eq!( + sealed.transitions.len(), + 100, + "all 100 casts carried a move → sparse set = 100 here" + ); + // A second drain of the same writer is empty (payload staging cleared). + let again = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); + assert!(again.is_empty(), "drain cleared the writer's staging"); + } + + // ── P4b HEADLINE FALSIFIER: 64k mailboxes / 17 sealed → exactly 17 advance ── + #[tokio::test] + async fn p4b_applies_only_the_sealed_sparse_set_64k_of_17_advance_rest_byte_identical() { + const FLEET: u32 = 65_536; + // 17 owners produce a material update; the other 65_519 do not. + let represented: Vec = (0..17).map(|i| i * 3_855).collect(); + let represented_set: HashSet = represented.iter().copied().collect(); + + let mut fleet: HashMap = (0..FLEET) + .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) + .collect(); + let before = fleet.clone(); + + let sink = FakeWalSink::new(); + let mut w = writer_with_moves(&represented); + let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); + let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + assert_eq!( + sealed.transitions.len(), + 17, + "sparse sealed set = 17, not 64k" + ); + + let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + + // Exactly the 17 represented owners advanced. + assert_eq!(applied.applied.len(), 17, "exactly 17 owners advanced"); + assert_eq!(applied.deferred, 0); + assert_eq!(applied.missing, 0); + assert_eq!(applied.version, DatasetVersion(1)); + + // Every represented owner is now at CognitiveWork (cycle bumped); every + // OTHER owner is byte-identical to its pre-cycle state. + let mut advanced_count = 0usize; + let mut untouched_count = 0usize; + for (id, owner) in &fleet { + if represented_set.contains(id) { + assert_eq!(owner.phase(), KanbanColumn::CognitiveWork); + assert_eq!(owner.current_cycle(), 1, "advanced owner bumped its cycle"); + advanced_count += 1; + } else { + assert_eq!( + owner, &before[id], + "unrepresented owner {id} must be BYTE-IDENTICAL" + ); + untouched_count += 1; + } + } + assert_eq!(advanced_count, 17); + assert_eq!( + untouched_count, + (FLEET as usize) - 17, + "anti-vacuity: the untouched set is 64k−17, not merely 'some'" + ); + + // Storage: exactly one WAL write, one version, and P4b read NO dataset. + assert_eq!(sink.wal_writes(), 1, "one WAL write for the whole sweep"); + assert_eq!( + sink.reads(), + 0, + "P4b reads no dataset (no second dataset read)" + ); + } + + // ── P4b: a DatasetVersion is NOT a fleet-step signal (empty sparse set) ────── + #[tokio::test] + async fn p4b_a_version_with_no_sealed_transitions_advances_nobody() { + // Every owner "participated" (a no-move cast) but NONE produced a move. + let mut fleet: HashMap = (0..1_000) + .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) + .collect(); + let before = fleet.clone(); + + let sink = FakeWalSink::new(); + let mut w: BatchWriter> = BatchWriter::new(); + for id in 0..1_000u32 { + w.cast(id, vec![], vec![0x00]); // no move → no transition + } + let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); + let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + assert!(sealed.transitions.is_empty(), "no moves → empty sparse set"); + + let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + assert!( + applied.applied.is_empty(), + "a version alone advances nobody" + ); + assert_eq!(fleet, before, "the whole fleet is byte-identical"); + assert_eq!(sink.wal_writes(), 1, "the cycle still sealed one version"); + } + + // ── P4b: interim one-transition-per-owner defers the rest ──────────────────── + #[tokio::test] + async fn p4b_defers_a_second_transition_for_the_same_owner() { + let mut fleet: HashMap = + HashMap::from([(42, FakeOwner::at(42, KanbanColumn::Planning))]); + let sink = FakeWalSink::new(); + // Two casts for owner 42 in one cycle. + let mut w: BatchWriter> = BatchWriter::new(); + w.cast( + 42, + vec![mv(42, KanbanColumn::Planning, KanbanColumn::CognitiveWork)], + vec![1], + ); + w.cast( + 42, + vec![mv( + 42, + KanbanColumn::CognitiveWork, + KanbanColumn::Evaluation, + )], + vec![2], + ); + let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); + let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + assert_eq!(sealed.transitions.len(), 2, "both cast a move"); + + let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + assert_eq!( + applied.applied.len(), + 1, + "≤1 durable transition per owner per cycle" + ); + assert_eq!(applied.deferred, 1, "the second transition is deferred"); + assert_eq!( + fleet[&42].phase(), + KanbanColumn::CognitiveWork, + "owner advanced exactly one step (the first in stream order)" + ); + } + + // ── P4b: a stale sealed move (from ≠ phase) is a corruption error ─────────── + #[tokio::test] + async fn p4b_stale_phase_is_a_corruption_error() { + // Owner is at CognitiveWork; the sealed move claims from=Planning. + let mut fleet: HashMap = + HashMap::from([(7, FakeOwner::at(7, KanbanColumn::CognitiveWork))]); + let sealed = SealedCycle { + version: DatasetVersion(1), + transitions: vec![SealedTransition { + stream_position: 0, + owner: 7, + mv: mv(7, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + }], + }; + assert!(matches!( + apply_sealed_transitions(&mut fleet, &sealed), + Err(PersistError::StalePhase { .. }) + )); + } + + // ── P4b: a sealed move for an unregistered owner is counted, not a crash ──── + #[tokio::test] + async fn p4b_missing_owner_is_counted_not_applied() { + let mut fleet: HashMap = HashMap::new(); // empty fleet + let sealed = SealedCycle { + version: DatasetVersion(1), + transitions: vec![SealedTransition { + stream_position: 0, + owner: 99, + mv: mv(99, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + }], + }; + let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + assert_eq!(applied.missing, 1); + assert!(applied.applied.is_empty()); + } + + // ── End-to-end: run_cycle closes P4a→P4b in one call ──────────────────────── + #[tokio::test] + async fn run_cycle_seals_then_applies_the_sparse_set() { + let mut fleet: HashMap = (0..10) + .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) + .collect(); + let sink = FakeWalSink::new(); + let mut w = writer_with_moves(&[3, 7]); // only owners 3 and 7 produce a move + + let (sealed, applied) = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + |id| u64::from(id), + ) + .await + .unwrap(); + + assert_eq!(sealed.version, DatasetVersion(1)); + assert_eq!(applied.applied.len(), 2, "exactly owners 3 and 7 advanced"); + assert_eq!(fleet[&3].phase(), KanbanColumn::CognitiveWork); + assert_eq!(fleet[&7].phase(), KanbanColumn::CognitiveWork); + assert_eq!( + fleet[&0].phase(), + KanbanColumn::Planning, + "owner 0 untouched" + ); + assert_eq!(sink.wal_writes(), 1); + assert_eq!(sink.reads(), 0); + } +} diff --git a/crates/lance-graph-supervisor/src/lib.rs b/crates/lance-graph-supervisor/src/lib.rs index 43871306..499bb907 100644 --- a/crates/lance-graph-supervisor/src/lib.rs +++ b/crates/lance-graph-supervisor/src/lib.rs @@ -40,6 +40,18 @@ pub mod consumer_msg; pub mod error; pub mod lifecycle_audit; +// ─── cycle-driver feature — the P4 loop-closure driver (persist_sink caller) ── + +/// P4a/P4b: the cycle loop-closure driver. Drains the fleet's staged planner +/// casts into one sealed cycle (`persist_cycle` → one WAL write, one +/// `DatasetVersion`) and applies ONLY the sealed **sparse** transition set to the +/// fleet — represented owners advance one legal step, unrepresented owners stay +/// byte-identical. Behind the `cycle-driver` feature (pulls `lance-graph-planner`); +/// it does NOT need the ractor `supervisor` feature — it applies through +/// `MailboxSoaOwner::try_advance_phase` directly, no message bus. +#[cfg(feature = "cycle-driver")] +pub mod cycle_driver; + pub use consumer_msg::{ CalibrateRequest, CalibrateResponse, ConsumerEnvelope, ConsumerReply, CrystalResponse, DispatchRequest, HealthStatus, IngestAck, IngestRequest, ProbeRequest, ProbeResponse, From fe54f5a843b1c9b0c6cb5367f75fc5f6f5c41ec2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 11:03:13 +0000 Subject: [PATCH 4/6] =?UTF-8?q?feat(supervisor):=20D-MBX-A6-P4c..P4f=20?= =?UTF-8?q?=E2=80=94=20complete=20the=20cycle=20loop-closure=20driver=20(s?= =?UTF-8?q?eal=E2=86=92step=E2=86=92think=E2=86=92cast=E2=86=92recover)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the P4a/P4b driver in `lance-graph-supervisor::cycle_driver` to the full control loop. Still mints NO domain types — reuses StrategyOutcome, owner_adapter::emit_bootstrap_intent, recover_and_apply, LandedSlot. - P4c `run_cognitive_work(fleet, applied, writer, think)`: owners that just entered CognitiveWork run a PLUGGABLE thought seam `think(&Owner) -> Option<(StrategyOutcome, payload)>` (NOT the shader) and route the Outcome into the next cycle's casts via owner_adapter (bootstrap-sentinel rebind → write-on-behalf cast). No mailbox mutation (the step is P4b, post-seal). MailboxFleet gained a read accessor `owner()`. - P4d wait-free: a completed owner casts + advances with no synchronous neighbour wait; an incomplete owner never blocks a completed one (structural — fire-and- forget cast, no per-owner barrier; the cycle boundary is the WAL-amortization barrier, not a neighbour wait). Proven by falsifier. - P4e `recover_fleet(sink, fleet, ids, watermarks)`: composes recover_and_apply per owner over scan_sealed; replays only the pending tail above each owner's durable watermark (idempotent); keeps the earned watermark on a mid-owner error. FleetRecovery{total_applied, owners_recovered}. - P4f sparse-routing scale probe: CountingFleet proves apply cost is O(dirty), not O(fleet) — 640 owner-resolutions over a 65_536-owner fleet. 11 cycle_driver lib tests green (--features cycle-driver): the 64k/17 headline + P4c round-trip (a CognitiveWork Outcome cast in cycle N advances the owner one further legal step in N+1) + P4d wait-free + P4e idempotence with a load-bearing- watermark NEGATIVE CONTROL (watermark lost → acyclic re-drive StalePhase-stalls) + P4f O(dirty). clippy clean on cycle_driver.rs, fmt clean; default (no-feature) supervisor build unchanged. The test FakeWalSink now stores landings for P4e. Durability leg still the contract-probe fake — control loop closed, storage NOT proven (Ladybug rule); concrete LanceShardSink deferred. The MedCare first-thought loop is code-complete on the control side. Remaining before a real first thought: (a) a concrete LanceShardSink (durability, gated on crash falsifiers) and (b) a real CognitiveWork thought body plugged into the P4c seam (the shader/StyleStrategy — exists). STATUS_BOARD D-MBX-A6-P4 → P4a–P4f Shipped (slice); LATEST_STATE prepended. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 10 + .claude/board/STATUS_BOARD.md | 2 +- .../src/cycle_driver.rs | 342 +++++++++++++++++- 3 files changed, 348 insertions(+), 6 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 9ec30f2a..82e016c3 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,13 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P4c..P4f: cycle loop-closure driver COMPLETE (slice) — the full seal→step→think→cast→recover loop + +- `lance-graph-supervisor::cycle_driver` extended P4a/P4b → **P4a–P4f** (feature `cycle-driver`; still mints NO domain types — reuses `StrategyOutcome`, `owner_adapter::emit_bootstrap_intent`, `recover_and_apply`, `LandedSlot`, `MailboxSoaOwner`): + - **P4c** — `run_cognitive_work(fleet, applied, writer, think)`: for each owner that just entered `CognitiveWork` (an applied move with `to == CognitiveWork`), run a **pluggable thought seam** `think(&Owner) -> Option<(StrategyOutcome, payload)>` (NOT the shader — designed elsewhere) and route its Outcome into the NEXT cycle's casts via `owner_adapter::emit_bootstrap_intent` (bootstrap-sentinel rebind → write-on-behalf cast). No mailbox mutation (the step is P4b, post-seal). `MailboxFleet` gained a read accessor `owner()`. + - **P4d** — wait-free: a completed owner casts + advances without any synchronous neighbour wait; an incomplete ("mid-thought") owner never blocks a completed one. Structural (fire-and-forget cast, no per-owner barrier — the cycle boundary is the WAL-amortization barrier, not a neighbour wait); proven by falsifier. + - **P4e** — `recover_fleet(sink, fleet, ids, watermarks)`: composes `persist_sink::recover_and_apply` per owner over `scan_sealed`, replays only the pending tail above each owner's durable **watermark** (idempotent); keeps the earned watermark on a mid-owner error (`Err((partial, cause))` contract). `FleetRecovery{total_applied, owners_recovered}`. + - **P4f** — sparse-routing scale probe: `CountingFleet` proves apply cost is **O(dirty), not O(fleet)** — 640 owner-resolutions over a 65 536-owner fleet (1% dirty), the sparse-cycle guarantee made measurable (`perf.p4f` log line). +- **11 cycle_driver lib tests green** (`cargo test -p lance-graph-supervisor --features cycle-driver`): the 64k/17 headline + P4c round-trip (a CognitiveWork Outcome cast in cycle N advances the owner one further legal step in N+1) + P4d wait-free + P4e idempotence with a **load-bearing-watermark negative control** (watermark lost → acyclic re-drive StalePhase-stalls) + P4f O(dirty). clippy (feature) clean on `cycle_driver.rs`, fmt clean; default (no-feature) supervisor build unchanged. Durability leg still the contract-probe fake (`FakeWalSink`, now storing landings for P4e) — control loop closed, storage NOT proven; concrete `LanceShardSink` deferred. +- **The MedCare first-thought loop is now code-complete on the control side:** seal→step→think→cast→recover all wired against the contract-probe sink. The only remaining pieces before a real first thought are (a) a concrete `LanceShardSink` (durability, gated on crash falsifiers) and (b) a real CognitiveWork thought body plugged into the P4c seam (the shader/StyleStrategy — exists; wiring is a consumer concern). STATUS_BOARD D-MBX-A6-P4 → P4a–P4f Shipped (slice). Plan `.claude/plans/cycle-loop-closure-driver-v1.md`. + ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P4a+P4b: cycle loop-closure driver (persist_sink's first production caller) - `lance-graph-supervisor::cycle_driver` (NEW module, behind feature **`cycle-driver`** = optional one-way `lance-graph-planner` dep; default supervisor build stays light, no planner/ractor) — **the seam that makes the merged #878 `persist_sink` load-bearing** (it was the ZERO-caller gap). Supervisor is the runtime fleet owner; planner decides; the dep is one-way (planner never deps supervisor — verified acyclic). Mints **NO** domain types — composes `SweepSlot`/`CycleFrame`/`CycleId`/`persist_cycle`/`WalSink`/`PersistError` (planner) + `BatchWriter` (planner) + `MailboxSoaOwner`/`KanbanMove`/`DatasetVersion`/`MailboxId` (contract). diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index f940a092..32d7d9ba 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -964,7 +964,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | | D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **Merged** | #878 (merged; reshaped in place to the cycle/WAL model = P3e — `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`; + the §2 sparse-delta storage ruling `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`, RATIFIED/UNIMPLEMENTED); ordering/recovery CONTRACT probed (348 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | -| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a+P4b Shipped (slice); P4c–f Queued** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** = `collect_casts` (drain `BatchWriter` → `Vec`) + `seal_cycle` (`persist_cycle` → one WAL write / one `DatasetVersion` + the sparse `SealedTransition` set). **P4b** = `apply_sealed_transitions` (iterate ONLY the sealed sparse set; each represented owner one legal `try_advance_phase`; unrepresented owners byte-identical; interim ≤1/owner/cycle → `deferred`; unknown owner → `missing`; reads NO dataset). `run_cycle` convenience. 7 lib tests incl. the **64k/17 falsifier** (65 536 mailboxes, 17 sealed → exactly 17 advance, 65 519 byte-identical, one WAL write, zero dataset reads); clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`MailboxSoaOwner`). Durability leg still the contract-probe fake. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | +| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a–P4f Shipped (slice)** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** `collect_casts`+`seal_cycle` (drain `BatchWriter` → `persist_cycle` → one WAL write / one `DatasetVersion` + sparse `SealedTransition` set). **P4b** `apply_sealed_transitions` (iterate ONLY the sealed sparse set; one legal `try_advance_phase`/represented owner; unrepresented byte-identical; interim ≤1/owner/cycle → `deferred`; unknown → `missing`; reads NO dataset). **P4c** `run_cognitive_work` (owners entering CognitiveWork run a pluggable thought seam → `owner_adapter::emit_bootstrap_intent` casts the next intent write-on-behalf → next cycle). **P4d** wait-free (an incomplete owner never blocks a completed one — no barrier). **P4e** `recover_fleet` (composes `persist_sink::recover_and_apply` per owner; per-owner watermark idempotence; partial-progress kept on error). **P4f** `CountingFleet` scale probe. `run_cycle` convenience. **11 lib tests** incl. the **64k/17 falsifier** (exactly 17 advance, 65 519 byte-identical, one WAL write, zero reads), P4c round-trip, P4d wait-free, P4e idempotence + load-bearing-watermark negative control, P4f **O(dirty)-not-O(fleet)** (640 resolves over a 64k fleet); clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`StrategyOutcome`/`recover_and_apply`/`emit_bootstrap_intent`/`MailboxSoaOwner`). CognitiveWork thought body is a **seam** (not the shader); durability leg still the contract-probe fake. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/crates/lance-graph-supervisor/src/cycle_driver.rs b/crates/lance-graph-supervisor/src/cycle_driver.rs index 23fb2852..b2fee029 100644 --- a/crates/lance-graph-supervisor/src/cycle_driver.rs +++ b/crates/lance-graph-supervisor/src/cycle_driver.rs @@ -44,14 +44,17 @@ use std::collections::{HashMap, HashSet}; use lance_graph_contract::collapse_gate::MailboxId; -use lance_graph_contract::kanban::KanbanMove; +use lance_graph_contract::kanban::{KanbanColumn, KanbanMove}; use lance_graph_contract::scheduler::DatasetVersion; use lance_graph_contract::soa_view::{MailboxSoaOwner, MailboxSoaView}; use lance_graph_planner::batch_writer::BatchWriter; +use lance_graph_planner::owner_adapter::emit_bootstrap_intent; use lance_graph_planner::persist_sink::{ - persist_cycle, CycleFrame, CycleId, PersistError, SweepSlot, WalSink, + persist_cycle, recover_and_apply, CycleFrame, CycleId, LandedSlot, PersistError, SweepSlot, + WalSink, }; +use lance_graph_planner::traits::StrategyOutcome; /// One sealed paired transition — a member of the **sparse** set a sealed cycle /// publishes. Carries `stream_position` so P4b applies transitions in canonical @@ -102,7 +105,9 @@ pub struct AppliedCycle { pub trait MailboxFleet { /// The concrete owner type this fleet holds. type Owner: MailboxSoaOwner; - /// Resolve a mailbox to its owner, or `None` if it is not registered. + /// Resolve a mailbox to its owner (read-only) — used by P4c's thought body. + fn owner(&self, id: MailboxId) -> Option<&Self::Owner>; + /// Resolve a mailbox to its owner for mutation, or `None` if not registered. fn owner_mut(&mut self, id: MailboxId) -> Option<&mut Self::Owner>; } @@ -110,6 +115,9 @@ pub trait MailboxFleet { /// production supervisor's owner registry and by the tests. impl MailboxFleet for HashMap { type Owner = O; + fn owner(&self, id: MailboxId) -> Option<&O> { + self.get(&id) + } fn owner_mut(&mut self, id: MailboxId) -> Option<&mut O> { self.get_mut(&id) } @@ -268,6 +276,117 @@ where Ok((sealed, applied)) } +/// **P4c.** For each owner that ENTERED `CognitiveWork` this cycle (an applied +/// move whose `to` is [`KanbanColumn::CognitiveWork`]), run the pluggable thought +/// body and route its Outcome into the NEXT cycle's casts via +/// `owner_adapter::emit_bootstrap_intent`. Returns the number of next-cycle +/// intents cast. +/// +/// The thought body is a **seam, not designed here** (§5.4 of the plan): `think` +/// is `FnMut(&Owner) -> Option<(StrategyOutcome, payload)>`. `None` = the thought +/// produced no next intent (the owner rests). The Outcome's `intended_move` must +/// be a **bootstrap sentinel** (`mailbox 0`) — `owner_adapter` rebinds it to the +/// live owner (no-theft) and casts it **write-on-behalf**, so the owner announces +/// where it is going and the next cycle collects it. This never mutates a mailbox +/// (the step is P4b, post-seal); it only stages the next intent. +pub fn run_cognitive_work( + fleet: &F, + applied: &AppliedCycle, + writer: &mut BatchWriter>, + mut think: impl FnMut(&F::Owner) -> Option<(StrategyOutcome, Vec)>, +) -> usize +where + F: MailboxFleet, +{ + let mut cast_count = 0usize; + for mv in &applied.applied { + // Only owners that just entered CognitiveWork run the thought body. + if mv.to != KanbanColumn::CognitiveWork { + continue; + } + let Some(owner) = fleet.owner(mv.mailbox) else { + continue; + }; + if let Some((outcome, payload)) = think(owner) { + // owner_adapter rebinds the bootstrap sentinel to this owner + casts it + // write-on-behalf; a non-sentinel / no-intent outcome stages nothing. + if emit_bootstrap_intent( + &outcome, + owner.mailbox_id(), + owner.current_cycle(), + writer, + payload, + ) + .is_some() + { + cast_count += 1; + } + } + } + cast_count +} + +/// The effect of a [`recover_fleet`] pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FleetRecovery { + /// Total moves re-applied across all recovered owners this pass. + pub total_applied: usize, + /// Owners that had a pending tail replayed (non-empty applied set). + pub owners_recovered: usize, +} + +/// **P4e.** Fleet-level crash recovery: scan the sealed landings once and replay +/// each owner's PENDING tail via `persist_sink::recover_and_apply`, idempotent +/// with a **per-owner watermark**. Only unreplayed moves (above the owner's +/// watermark) are applied; already-applied moves are skipped; unrepresented +/// owners are untouched. `watermarks` is updated in place with the new per-owner +/// watermark to persist alongside the SoA phase. +/// +/// On a mid-owner failure the partial progress is kept: the failing owner's +/// watermark is still advanced for its applied prefix (per +/// `recover_and_apply`'s `Err((partial, cause))` contract) before the error is +/// returned, so a re-drive does not replay the applied prefix. +pub async fn recover_fleet( + sink: &S, + fleet: &mut F, + fleet_ids: &[MailboxId], + watermarks: &mut HashMap>, +) -> Result +where + S: WalSink, + F: MailboxFleet, +{ + let sealed: Vec = sink.scan_sealed(None).await.map_err(PersistError::Write)?; + let mut total_applied = 0usize; + let mut owners_recovered = 0usize; + for &id in fleet_ids { + let Some(owner) = fleet.owner_mut(id) else { + continue; + }; + let wm = watermarks.get(&id).copied().flatten(); + match recover_and_apply(owner, &sealed, wm) { + Ok(rec) => { + if !rec.applied.is_empty() { + owners_recovered += 1; + } + total_applied += rec.applied.len(); + watermarks.insert(id, rec.watermark); + } + Err((partial, cause)) => { + // Keep the earned watermark for the applied prefix, then surface the + // error (the partial `applied` count is discarded — the caller + // re-drives from the persisted watermark). + watermarks.insert(id, partial.watermark); + return Err(cause); + } + } + } + Ok(FleetRecovery { + total_applied, + owners_recovered, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -341,6 +460,7 @@ mod tests { struct SealedRec { frame: CycleFrame, version: DatasetVersion, + landings: Vec, } struct FakeWalSink { sealed: Mutex>, @@ -380,15 +500,28 @@ mod tests { sealed.push(SealedRec { frame: batch.frame, version, + landings: batch.landings, }); Ok(version) } async fn scan_sealed( &self, - _from: Option, + from: Option, ) -> Result, WriteFailed> { self.reads.fetch_add(1, Ordering::SeqCst); - Ok(Vec::new()) + Ok(self + .sealed + .lock() + .unwrap() + .iter() + .filter(|s| from.is_none_or(|f| s.version > f)) + .flat_map(|s| { + s.landings.iter().map(|slot| LandedSlot { + version: s.version, + slot: slot.clone(), + }) + }) + .collect()) } async fn versions(&self) -> Result, WriteFailed> { self.reads.fetch_add(1, Ordering::SeqCst); @@ -655,4 +788,203 @@ mod tests { assert_eq!(sink.wal_writes(), 1); assert_eq!(sink.reads(), 0); } + + /// A bootstrap-sentinel move (owner 0, cycle 0) that `owner_adapter` rebinds. + fn sentinel(from: KanbanColumn, to: KanbanColumn) -> KanbanMove { + KanbanMove { + mailbox: 0, + from, + to, + witness_chain_position: 0, + exec: ExecTarget::Native, + } + } + + // ── P4c FALSIFIER: CognitiveWork thought → next-cycle cast → round-trip ───── + #[tokio::test] + async fn p4c_cognitive_work_casts_the_next_intent_and_round_trips() { + let sink = FakeWalSink::new(); + let mut fleet: HashMap = + HashMap::from([(5, FakeOwner::at(5, KanbanColumn::Planning))]); + let mut w = writer_with_moves(&[5]); + + // Cycle 1: owner 5 casts Planning→CognitiveWork; the driver applies it. + let (_s1, applied1) = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + u64::from, + ) + .await + .unwrap(); + assert_eq!(fleet[&5].phase(), KanbanColumn::CognitiveWork); + + // P4c: owner 5 (now in CognitiveWork) thinks → intends CognitiveWork→Evaluation, + // cast as a bootstrap sentinel into the writer for cycle 2. + let cast_count = run_cognitive_work(&fleet, &applied1, &mut w, |owner| { + assert_eq!(owner.phase(), KanbanColumn::CognitiveWork); + let outcome = StrategyOutcome { + reliability: 0.9, + intended_move: Some(sentinel( + KanbanColumn::CognitiveWork, + KanbanColumn::Evaluation, + )), + }; + Some((outcome, vec![0xCC])) + }); + assert_eq!(cast_count, 1, "one next-cycle intent cast"); + + // Cycle 2: the driver drains that cast → seals V2 → applies → owner 5 → Evaluation. + let (s2, applied2) = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(2), DatasetVersion(1)), + u64::from, + ) + .await + .unwrap(); + assert_eq!(s2.version, DatasetVersion(2)); + assert_eq!( + applied2.applied.len(), + 1, + "the round-tripped intent advanced owner 5 one further step" + ); + assert_eq!(fleet[&5].phase(), KanbanColumn::Evaluation); + } + + // ── P4d FALSIFIER: an incomplete owner never blocks a completed one ───────── + #[tokio::test] + async fn p4d_an_incomplete_owner_never_blocks_a_completed_one() { + // Owner A(1) completes + casts; owner B(2) is "mid-thought" (no cast). + let sink = FakeWalSink::new(); + let mut fleet: HashMap = HashMap::from([ + (1, FakeOwner::at(1, KanbanColumn::Planning)), + (2, FakeOwner::at(2, KanbanColumn::Planning)), + ]); + let mut w = writer_with_moves(&[1]); // ONLY A casts + + let (_s, applied) = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + u64::from, + ) + .await + .unwrap(); + + // A advanced without waiting for B; B is byte-identical. No barrier, no error. + assert_eq!(applied.applied.len(), 1); + assert_eq!( + fleet[&1].phase(), + KanbanColumn::CognitiveWork, + "A completed + advanced" + ); + assert_eq!( + fleet[&2].phase(), + KanbanColumn::Planning, + "B mid-thought never blocked A" + ); + assert_eq!( + fleet[&2].current_cycle(), + 0, + "B byte-identical — no neighbour wait" + ); + } + + // ── P4e FALSIFIER: recovery replays the pending tail, idempotent w/ watermark ─ + #[tokio::test] + async fn p4e_recover_fleet_replays_pending_tail_idempotent_with_watermark() { + // Seal a cycle with owner 5's Planning→CognitiveWork move (a durable landing). + let sink = FakeWalSink::new(); + let mut w = writer_with_moves(&[5]); + let casts = collect_casts(&mut w, CycleId(1), u64::from); + seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + + // Restart: a FRESH owner 5 at its pre-move phase, empty watermarks. + let mut fleet: HashMap = + HashMap::from([(5, FakeOwner::at(5, KanbanColumn::Planning))]); + let mut wm: HashMap> = HashMap::new(); + + let rec = recover_fleet(&sink, &mut fleet, &[5], &mut wm) + .await + .unwrap(); + assert_eq!(rec.total_applied, 1, "the pending move was replayed"); + assert_eq!(rec.owners_recovered, 1); + assert_eq!(fleet[&5].phase(), KanbanColumn::CognitiveWork); + + // Re-drive with the returned watermark → idempotent (nothing re-applied). + let again = recover_fleet(&sink, &mut fleet, &[5], &mut wm) + .await + .unwrap(); + assert_eq!( + again.total_applied, 0, + "watermark makes recovery idempotent" + ); + + // Negative control: watermark LOST → re-driving the already-advanced owner + // stalls (from=Planning ≠ phase=CognitiveWork) → the watermark is load-bearing. + let mut wm_lost: HashMap> = HashMap::new(); + let stalled = recover_fleet(&sink, &mut fleet, &[5], &mut wm_lost).await; + assert!( + matches!(stalled, Err(PersistError::StalePhase { .. })), + "without the watermark an acyclic re-drive stalls — watermark is load-bearing" + ); + } + + // A fleet that COUNTS owner resolutions — proves apply cost is O(sparse set). + struct CountingFleet { + inner: HashMap, + resolves: usize, + } + impl MailboxFleet for CountingFleet { + type Owner = FakeOwner; + fn owner(&self, id: MailboxId) -> Option<&FakeOwner> { + self.inner.get(&id) + } + fn owner_mut(&mut self, id: MailboxId) -> Option<&mut FakeOwner> { + self.resolves += 1; + self.inner.get_mut(&id) + } + } + + // ── P4f (SCALE): apply cost scales with the SPARSE set, not the fleet ─────── + #[tokio::test] + async fn p4f_apply_cost_scales_with_the_sparse_set_not_the_fleet() { + const FLEET: u32 = 65_536; + const DIRTY: u32 = 640; // ~1% sparse dirty fraction + let inner: HashMap = (0..FLEET) + .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) + .collect(); + let mut fleet = CountingFleet { inner, resolves: 0 }; + + let sink = FakeWalSink::new(); + let represented: Vec = (0..DIRTY).map(|i| i * 100).collect(); + let mut w = writer_with_moves(&represented); + let casts = collect_casts(&mut w, CycleId(1), u64::from); + let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + assert_eq!(sealed.transitions.len(), DIRTY as usize); + + let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + assert_eq!( + applied.applied.len(), + DIRTY as usize, + "exactly the dirty set advanced" + ); + // THE MEASUREMENT: owner resolution touched the sparse set ONLY — 640, not 64k. + assert_eq!( + fleet.resolves, DIRTY as usize, + "apply resolves exactly the sparse set (O(dirty)), never the {FLEET}-owner fleet" + ); + eprintln!( + "perf.p4f fleet={FLEET} dirty={DIRTY} resolves={} (sparse routing: apply is O(dirty), not O(fleet))", + fleet.resolves + ); + } } From c2134c159e7bc7467e27b07920b922bea8531bdc Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 11:16:13 +0000 Subject: [PATCH 5/6] supervisor(cycle_driver): plug the real MUL gate into the P4c seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The P4c CognitiveWork thought body was a pluggable seam; wire the REAL shader — the MUL cognitive gate — into it, minting no new decision logic. - shade_owner(owner, qualia, mantissa, reliability): reads the owner's current phase, runs contract::mul::i4_eval::gate_decision_i4 (the i4 TrustTexture x FlowState gate), lowers via KanbanColumn::advance_on_gate (Flow->forward, Block->Prune-where-legal, Hold->rest). Composes kanban_actor::mul_target for the driver. Returns a bootstrap-sentinel StrategyOutcome so owner_adapter rebinds + casts write-on-behalf; no mailbox mutated (the durable step is P4b, next cycle). - run_cognitive_work_gated: the shader-wired form of run_cognitive_work; a caller-supplied read_gate extractor supplies (qualia, mantissa, reliability, payload) and the gate decides. Delegates to run_cognitive_work (single routing path). - The qualia seam: MailboxSoaView does not yet expose qualia() (deferred); P4c is the first consumer, so the extractor bridges it without touching the trait — the MailboxSoa contract stays unchanged. Tests: +3 (11 -> 14). shade_owner Flow/Block/Hold discriminate (three distinct outputs), absorbing-column yields None (DAG respected), and a gated round-trip proves a Flow-qualia owner casts + advances to Evaluation next cycle while a Hold-qualia owner rests at CognitiveWork. clippy+fmt clean; default build unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 9 + .claude/board/STATUS_BOARD.md | 2 +- .../src/cycle_driver.rs | 203 +++++++++++++++++- 3 files changed, 212 insertions(+), 2 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 82e016c3..7e6d2314 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,12 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P4c shader plug: the REAL MUL cognitive gate wired into the CognitiveWork seam + +- `lance-graph-supervisor::cycle_driver` gained the **shader plug** — the P4c thought body is no longer just a pluggable seam, it is the REAL MUL cognitive gate (mints NO decision logic; it composes `kanban_actor::mul_target` for the driver from two already-shipped contract primitives): + - **`shade_owner(owner, qualia, mantissa, reliability) -> Option`** — reads the owner's current phase → `contract::mul::i4_eval::gate_decision_i4(qualia, mantissa)` (the i4 TrustTexture × FlowState gate: Flow / Hold / Block) → lowers via `KanbanColumn::advance_on_gate` (Flow → forward, Block → Prune-where-legal, Hold → rest / `None`). Packages the result as a **bootstrap-sentinel** `StrategyOutcome` (`mailbox 0`, `witness_chain_position 0`) so `owner_adapter::emit_bootstrap_intent` rebinds it to the live owner and casts write-on-behalf — **no mailbox mutated** (the step is P4b, next cycle). + - **`run_cognitive_work_gated(fleet, applied, writer, read_gate)`** — the shader-wired form of `run_cognitive_work`: for each owner entering `CognitiveWork`, `read_gate(&Owner) -> Option<(QualiaI4_16D, i8, f32, payload)>` supplies the gate inputs and the gate decides the next move. Delegates to `run_cognitive_work` (single routing path). + - **The qualia seam (no MailboxSoa redesign):** `MailboxSoaView` does NOT yet expose `qualia()` (deferred — `soa_view.rs` "add `fn qualia` when the first consumer arrives"). P4c is that first consumer; the caller-supplied extractor bridges the seam so the **MailboxSoa contract stays UNCHANGED** (operator constraint "do not redesign the MailboxSoa" respected — the trait method lands later without touching this code). +- **14 cycle_driver lib tests green** (was 11; +3): `shade_owner_flow_advances_forward_block_prunes_hold_rests` (three distinct outputs Flow→Evaluation / Block→Prune / Hold→None — the gate discriminates, per the anti-eigenvalue falsifiability rule), `shade_owner_at_absorbing_column_yields_nothing` (Flow/Block at Commit → `None`, DAG respected), and `run_cognitive_work_gated_flow_casts_next_intent_hold_casts_nothing` (a Flow-qualia owner casts + round-trips to Evaluation in the next cycle while a Hold-qualia owner rests at CognitiveWork — one WAL write per cycle, sparse). clippy (feature) clean on `cycle_driver.rs`, fmt clean; default build unchanged. +- **The MedCare first-thought loop now runs the real gate on the control side:** seal→step→**think (real MUL gate)**→cast→recover, all against the contract-probe sink. The ONE remaining fake is the durability leg (`FakeWalSink`); the concrete `LanceShardSink` is still deferred (gated on crash falsifiers). STATUS_BOARD D-MBX-A6-P4 updated in place. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`. + ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P4c..P4f: cycle loop-closure driver COMPLETE (slice) — the full seal→step→think→cast→recover loop - `lance-graph-supervisor::cycle_driver` extended P4a/P4b → **P4a–P4f** (feature `cycle-driver`; still mints NO domain types — reuses `StrategyOutcome`, `owner_adapter::emit_bootstrap_intent`, `recover_and_apply`, `LandedSlot`, `MailboxSoaOwner`): diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 32d7d9ba..d8156727 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -964,7 +964,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | | D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **Merged** | #878 (merged; reshaped in place to the cycle/WAL model = P3e — `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`; + the §2 sparse-delta storage ruling `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`, RATIFIED/UNIMPLEMENTED); ordering/recovery CONTRACT probed (348 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | -| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a–P4f Shipped (slice)** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** `collect_casts`+`seal_cycle` (drain `BatchWriter` → `persist_cycle` → one WAL write / one `DatasetVersion` + sparse `SealedTransition` set). **P4b** `apply_sealed_transitions` (iterate ONLY the sealed sparse set; one legal `try_advance_phase`/represented owner; unrepresented byte-identical; interim ≤1/owner/cycle → `deferred`; unknown → `missing`; reads NO dataset). **P4c** `run_cognitive_work` (owners entering CognitiveWork run a pluggable thought seam → `owner_adapter::emit_bootstrap_intent` casts the next intent write-on-behalf → next cycle). **P4d** wait-free (an incomplete owner never blocks a completed one — no barrier). **P4e** `recover_fleet` (composes `persist_sink::recover_and_apply` per owner; per-owner watermark idempotence; partial-progress kept on error). **P4f** `CountingFleet` scale probe. `run_cycle` convenience. **11 lib tests** incl. the **64k/17 falsifier** (exactly 17 advance, 65 519 byte-identical, one WAL write, zero reads), P4c round-trip, P4d wait-free, P4e idempotence + load-bearing-watermark negative control, P4f **O(dirty)-not-O(fleet)** (640 resolves over a 64k fleet); clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`StrategyOutcome`/`recover_and_apply`/`emit_bootstrap_intent`/`MailboxSoaOwner`). CognitiveWork thought body is a **seam** (not the shader); durability leg still the contract-probe fake. Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | +| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a–P4f Shipped (slice)** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** `collect_casts`+`seal_cycle` (drain `BatchWriter` → `persist_cycle` → one WAL write / one `DatasetVersion` + sparse `SealedTransition` set). **P4b** `apply_sealed_transitions` (iterate ONLY the sealed sparse set; one legal `try_advance_phase`/represented owner; unrepresented byte-identical; interim ≤1/owner/cycle → `deferred`; unknown → `missing`; reads NO dataset). **P4c** `run_cognitive_work` (owners entering CognitiveWork run a pluggable thought seam → `owner_adapter::emit_bootstrap_intent` casts the next intent write-on-behalf → next cycle) **+ shader plug** `shade_owner`/`run_cognitive_work_gated` (the REAL MUL gate: `contract::mul::i4_eval::gate_decision_i4(qualia,mantissa)` → `KanbanColumn::advance_on_gate` — Flow→forward, Block→Prune-where-legal, Hold→rest — packaged as a bootstrap-sentinel `StrategyOutcome`; `mul_target` composed for the driver, mints no decision logic; `qualia`/`mantissa` via a caller extractor bridging the deferred `MailboxSoaView::qualia()` seam — MailboxSoa contract UNCHANGED). **P4d** wait-free (an incomplete owner never blocks a completed one — no barrier). **P4e** `recover_fleet` (composes `persist_sink::recover_and_apply` per owner; per-owner watermark idempotence; partial-progress kept on error). **P4f** `CountingFleet` scale probe. `run_cycle` convenience. **14 lib tests** incl. the **64k/17 falsifier** (exactly 17 advance, 65 519 byte-identical, one WAL write, zero reads), P4c round-trip, the **shader-gate falsifier** (Flow-qualia owner casts + advances to Evaluation while a Hold-qualia owner rests at CognitiveWork — the gate discriminates, three distinct outputs Flow/Hold/Block for three inputs, plus absorbing-column no-successor), P4d wait-free, P4e idempotence + load-bearing-watermark negative control, P4f **O(dirty)-not-O(fleet)** (640 resolves over a 64k fleet); clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`StrategyOutcome`/`recover_and_apply`/`emit_bootstrap_intent`/`MailboxSoaOwner`). CognitiveWork thought body is now the REAL MUL gate (`shade_owner`) via the caller-supplied qualia extractor; the only remaining fake is the durability leg (contract-probe `WalSink`, until `LanceShardSink`). Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/crates/lance-graph-supervisor/src/cycle_driver.rs b/crates/lance-graph-supervisor/src/cycle_driver.rs index b2fee029..927b5101 100644 --- a/crates/lance-graph-supervisor/src/cycle_driver.rs +++ b/crates/lance-graph-supervisor/src/cycle_driver.rs @@ -44,9 +44,11 @@ use std::collections::{HashMap, HashSet}; use lance_graph_contract::collapse_gate::MailboxId; -use lance_graph_contract::kanban::{KanbanColumn, KanbanMove}; +use lance_graph_contract::kanban::{ExecTarget, KanbanColumn, KanbanMove}; +use lance_graph_contract::mul::i4_eval::gate_decision_i4; use lance_graph_contract::scheduler::DatasetVersion; use lance_graph_contract::soa_view::{MailboxSoaOwner, MailboxSoaView}; +use lance_graph_contract::QualiaI4_16D; use lance_graph_planner::batch_writer::BatchWriter; use lance_graph_planner::owner_adapter::emit_bootstrap_intent; @@ -326,6 +328,78 @@ where cast_count } +/// **The shader plug (P4c gate).** Run the real MUL cognitive gate over an owner +/// that just entered `CognitiveWork` and lower the decision to the owner's next +/// intended move — the "thinking" the P4c seam routes. This is +/// `kanban_actor::mul_target` composed for the driver: it mints **no** new +/// decision logic, it reuses the two shipped contract primitives. +/// +/// 1. read the owner's *current* phase, +/// 2. run [`gate_decision_i4`]`(qualia, mantissa)` — the i4 TrustTexture × FlowState +/// gate (`Flow` / `Hold` / `Block`), +/// 3. lower the `GateDecision` to a DAG-legal next phase via +/// [`KanbanColumn::advance_on_gate`] (`Flow` → forward, `Block` → +/// Prune-where-legal, `Hold` → rest). +/// +/// The result is packaged as a **bootstrap sentinel** [`StrategyOutcome`] +/// (`mailbox 0`, `witness_chain_position 0`) so +/// `owner_adapter::emit_bootstrap_intent` rebinds it to the live owner and casts +/// it write-on-behalf — **no mailbox is mutated here** (the durable step is P4b, +/// next cycle). Returns `None` when the gate **Holds** (or yields no legal +/// successor): the owner rests this cycle and casts nothing. +/// +/// `qualia` + `mantissa` are supplied by the caller because `MailboxSoaView` does +/// **not yet** expose `qualia()` (deferred — `soa_view.rs` "add `fn qualia` when +/// the first consumer arrives"). P4c is that first consumer; until the trait +/// method lands, a fleet-specific extractor bridges the seam. This keeps the +/// MailboxSoa contract UNCHANGED — no trait redesign. +#[must_use] +pub fn shade_owner( + owner: &O, + qualia: &QualiaI4_16D, + mantissa: i8, + reliability: f32, +) -> Option { + let phase = owner.phase(); + let gate = gate_decision_i4(qualia, mantissa); + let to = phase.advance_on_gate(&gate)?; + Some(StrategyOutcome { + reliability, + intended_move: Some(KanbanMove { + // bootstrap sentinel — owner_adapter rebinds mailbox 0 to the live owner. + mailbox: 0, + from: phase, + to, + witness_chain_position: 0, + exec: ExecTarget::Native, + }), + }) +} + +/// **P4c with the real shader wired in.** Like [`run_cognitive_work`], but the +/// thought body IS the MUL cognitive gate ([`shade_owner`]) rather than a +/// caller-supplied Outcome. For each owner that just entered `CognitiveWork`, +/// `read_gate` extracts that owner's `(qualia, signed_mantissa, reliability, +/// payload)` — the qualia seam the deferred `MailboxSoaView::qualia()` will +/// eventually close — and the gate decides the next move. A **Hold** (or an owner +/// `read_gate` declines with `None`) casts nothing. Returns the number of +/// next-cycle intents cast. +pub fn run_cognitive_work_gated( + fleet: &F, + applied: &AppliedCycle, + writer: &mut BatchWriter>, + mut read_gate: impl FnMut(&F::Owner) -> Option<(QualiaI4_16D, i8, f32, Vec)>, +) -> usize +where + F: MailboxFleet, +{ + run_cognitive_work(fleet, applied, writer, |owner| { + let (qualia, mantissa, reliability, payload) = read_gate(owner)?; + let outcome = shade_owner(owner, &qualia, mantissa, reliability)?; + Some((outcome, payload)) + }) +} + /// The effect of a [`recover_fleet`] pass. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FleetRecovery { @@ -987,4 +1061,131 @@ mod tests { fleet.resolves ); } + + // ── The shader plug (P4c gate) ────────────────────────────────────────────── + + /// Flow qualia (warmth=4, groundedness=3, coherence=4, valence=2) — the same + /// construction `kanban_actor::s2_driver_gate_advances_then_holds` uses: + /// `flow_proxy = 4+3−0 = 7 ≥ 4` + mantissa>0 → FlowState::Flow; coherence≥4 + + /// valence≥2 + tension≤1 → TrustTexture::Calibrated ⇒ gate `Flow`. + fn flow_qualia() -> QualiaI4_16D { + QualiaI4_16D(0).with(3, 4).with(14, 3).with(9, 4).with(1, 2) + } + + /// Uncertain qualia (coherence=−3, tension=3) ⇒ TrustTexture::Uncertain ⇒ + /// gate `Block`. + fn block_qualia() -> QualiaI4_16D { + QualiaI4_16D(0).with(9, -3).with(2, 3) + } + + // shade_owner is the REAL gate: Flow→forward, Block→Prune, Hold→None. Three + // distinct outputs for three distinct inputs — the gate discriminates (it is + // not a constant that always fires the same way). + #[test] + fn shade_owner_flow_advances_forward_block_prunes_hold_rests() { + // Flow at CognitiveWork → forward to Evaluation (the only non-Prune next). + let cw = FakeOwner::at(1, KanbanColumn::CognitiveWork); + let out = shade_owner(&cw, &flow_qualia(), 4, 0.9).expect("Flow yields a move"); + let m = out.intended_move.expect("Flow carries an intended move"); + assert_eq!(m.from, KanbanColumn::CognitiveWork); + assert_eq!(m.to, KanbanColumn::Evaluation, "Flow → forward"); + assert_eq!( + m.mailbox, 0, + "bootstrap sentinel — owner_adapter rebinds it" + ); + assert_eq!(m.witness_chain_position, 0, "sentinel witness position"); + assert!((out.reliability - 0.9).abs() < f32::EPSILON); + + // Block at Planning → Prune (the Prune-where-legal branch). + let plan = FakeOwner::at(2, KanbanColumn::Planning); + let out = shade_owner(&plan, &block_qualia(), -4, 0.5).expect("Block yields a Prune"); + assert_eq!( + out.intended_move.unwrap().to, + KanbanColumn::Prune, + "Block → Prune-where-legal" + ); + + // Hold (neutral qualia, mantissa 0) → None: the owner rests, casts nothing. + assert!( + shade_owner(&cw, &QualiaI4_16D(0), 0, 0.5).is_none(), + "Hold must not produce a move" + ); + } + + // shade_owner respects the DAG: Block at an absorbing column (Commit) has no + // legal successor (`next_phases` empty) → None, even though the gate said Block. + #[test] + fn shade_owner_at_absorbing_column_yields_nothing() { + let done = FakeOwner::at(3, KanbanColumn::Commit); + assert!( + shade_owner(&done, &flow_qualia(), 4, 1.0).is_none(), + "Flow at Commit has no forward successor" + ); + assert!( + shade_owner(&done, &block_qualia(), -4, 1.0).is_none(), + "Block at Commit has no Prune successor" + ); + } + + // ── P4c GATED FALSIFIER: Flow-qualia thought → next cast → round-trip ──────── + #[tokio::test] + async fn run_cognitive_work_gated_flow_casts_next_intent_hold_casts_nothing() { + let sink = FakeWalSink::new(); + // Owner 5 will FLOW (advances); owner 6 will HOLD (rests). + let mut fleet: HashMap = HashMap::from([ + (5, FakeOwner::at(5, KanbanColumn::Planning)), + (6, FakeOwner::at(6, KanbanColumn::Planning)), + ]); + let mut w = writer_with_moves(&[5, 6]); + + // Cycle 1: both cast Planning→CognitiveWork; the driver applies both. + let (_s1, applied1) = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + u64::from, + ) + .await + .unwrap(); + assert_eq!(applied1.applied.len(), 2); + assert_eq!(fleet[&5].phase(), KanbanColumn::CognitiveWork); + assert_eq!(fleet[&6].phase(), KanbanColumn::CognitiveWork); + + // P4c: the REAL gate runs. Owner 5 gets Flow qualia → casts + // CognitiveWork→Evaluation. Owner 6 gets neutral qualia → Hold → no cast. + let cast_count = run_cognitive_work_gated(&fleet, &applied1, &mut w, |owner| { + let payload = vec![owner.mailbox_id() as u8]; + if owner.mailbox_id() == 5 { + Some((flow_qualia(), 4, 0.9, payload)) // FLOW + } else { + Some((QualiaI4_16D(0), 0, 0.5, payload)) // HOLD + } + }); + assert_eq!(cast_count, 1, "only the Flow owner cast a next intent"); + + // Cycle 2: the driver drains that single cast → seals V2 → applies → + // owner 5 → Evaluation; owner 6 stayed at CognitiveWork (it Held). + let (s2, applied2) = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(2), DatasetVersion(1)), + u64::from, + ) + .await + .unwrap(); + assert_eq!(s2.version, DatasetVersion(2)); + assert_eq!(applied2.applied.len(), 1, "only the Flow owner advanced"); + assert_eq!( + fleet[&5].phase(), + KanbanColumn::Evaluation, + "Flow owner advanced one further step through the real gate" + ); + assert_eq!( + fleet[&6].phase(), + KanbanColumn::CognitiveWork, + "Hold owner rested — the gate discriminates, it is not a constant" + ); + } } From 71d1db1e0f39b74959c0514c22a55e7efccf5b5b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 11:56:11 +0000 Subject: [PATCH 6/6] =?UTF-8?q?supervisor(cycle=5Fdriver):=20review=20roun?= =?UTF-8?q?d=20=E2=80=94=20recovery/data-integrity=20fixes=20+=20scope=20h?= =?UTF-8?q?onesty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grain-of-salt review of PR #879; each finding verified against code before acting. All accepted items were real defects: - Retry-safe seal: seal_cycle -> Result>; a WAL failure returns the complete frozen cast set byte-identical for retry (previously the drained cycle was simply lost). Falsifier: failed commit -> zero owner mutation -> same-cycle retry -> one version. - Restart-stable stream positions: collect_casts now takes the caller's durable position_base cursor; SealedCycle.next_position_base carries it forward (computed over ALL slots). Raw CastId was the P3d-documented 'cast_id is provenance only' trap: a reconstructed BatchWriter restarts at 0 and recover_and_apply silently skips positions <= watermark. Restart falsifier pins the exact failure mode. - Watermark-coupled apply: apply_sealed_transitions advances the per-owner recovery watermark WITH the phase (one rule shared with recovery); a crash after normal apply no longer replays into a StalePhase stall. - <=1-move/owner enforced PRE-seal: extras (same cast or later casts; also fixes the silent moves.first() truncation) return as HeldIntent, re-staged via restage_held into a future cycle. Sealed set == applied set, so recovery and normal operation agree; the old seal-then-defer counter (which discarded durable moves) is demoted to defence-in-depth. - Mid-apply errors return the applied prefix Err((partial, cause)), mirroring recover_and_apply. - Hold = reschedule, never strand: CognitiveWorkOutcome.held_owners + run_cognitive_work[_gated]_over re-poll; falsifier wakes a Held owner. - recover_fleet partitions sealed history once (O(history), not O(fleet x history)). - Scope honesty: module honesty ledger (control-loop contract proven; actor-owned production wiring NOT proven — MailboxFleet HashMap is the probe/registry fleet, KanbanActor bridging open; shader-driver/SoA thought NOT proven — the MUL gate is real, inputs extractor-fed; durability fake). P4d reworded to wait-free-at-the-cast-boundary with a two-represented-owners falsifier (A unfinished, B casts regardless). Declined: routing P4b through KanbanActor mailboxes — contradicts the ratified writer-fires-inline sparse ruling; the honesty half is taken in docs instead. 19 lib tests green (was 14); clippy + fmt clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 15 + .claude/board/STATUS_BOARD.md | 2 +- .claude/plans/cycle-loop-closure-driver-v1.md | 21 +- .../src/cycle_driver.rs | 1127 +++++++++++++---- 4 files changed, 904 insertions(+), 261 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 7e6d2314..ada1f2ff 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,18 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — PR #879 review round: recovery/data-integrity holes fixed + scope honesty (grain-of-salt audit) + +Operator-forwarded review (grain of salt); each finding verified against code before acting. **Accepted + fixed (all real):** + +1. **Retry-safe seal.** `collect_casts` drained the writer, `seal_cycle` consumed the casts — a WAL failure LOST the cycle (retry saw an empty writer). Now `seal_cycle → Result>` where `SealFailure{frame, casts, cause}` carries the complete frozen input back byte-identical; falsifier proves failed-commit → zero owner mutation → same-cycle retry → exactly one version, no cast lost/duplicated. +2. **Restart-stable stream positions.** `stream_position = cast.0` walked into the P3d-documented trap ("cast_id is provenance only" — a reconstructed `BatchWriter` restarts at 0, and `recover_and_apply` skips positions ≤ watermark → later cycles silently unrecoverable). Now `collect_casts(writer, cycle, position_base, row_of)` — `position_base` is the caller's DURABLE cursor; `SealedCycle.next_position_base` (computed over ALL slots incl. no-move landings) carries it forward. Restart falsifier pins the exact failure the raw-CastId scheme would have caused. +3. **Normal apply advances the recovery watermark.** `apply_sealed_transitions(fleet, sealed, &mut watermarks)` now moves phase + per-owner watermark TOGETHER (one owner state, the same rule recovery uses). Falsifier: normal apply → crash → `recover_fleet` with the same map replays NOTHING (previously: replay → permanent `StalePhase` stall; our own P4e negative control had proven the stall and we shipped the gap anyway). +4. **≤1-move/owner enforced PRE-seal.** The old `deferred += 1` sealed a durable move and then never applied it (and recovery WOULD apply it — divergent semantics). Now `collect_casts` partitions: first move per owner seals; every extra move (same cast or later cast — also killing the silent `moves.first()` truncation) returns as `HeldIntent`, re-staged via `restage_held` into a future cycle. Sealed set == applied set; recovery-agrees falsifier. `AppliedCycle.deferred` demoted to defence-in-depth for foreign sealed inputs. +5. **Mid-apply prefix preserved.** `apply_sealed_transitions` → `Err((partial, cause))` (mirrors `recover_and_apply`): the applied prefix + its watermarks survive a guard trip. +6. **Hold = reschedule, never strand.** `CognitiveWorkOutcome{cast, held_owners}`; held owners re-polled via `run_cognitive_work[_gated]_over`. Falsifier: a Held owner is woken on a later re-poll and advances. +7. **`recover_fleet` partitions history once** — O(history + Σtails), not O(fleet×history). +8. **P4d honesty**: wait-free at the cast/cycle boundary (sequential pass); strengthened falsifier: TWO represented owners, A unfinished, B casts + advances without waiting. Concurrent per-owner execution = the actor leg, explicitly out of driver scope. + +**Declined (with reasons):** routing P4b through `KanbanActor` mailboxes — contradicts the operator-ratified writer-fires-inline sparse ruling (no message bus for P4a/P4b; the plan's explicit shape). The honesty half IS taken: module docs now state `MailboxFleet`+HashMap is the probe/registry fleet, NOT production supervisor ownership; actor-state bridging is open. **Scope honesty ledger (module doc + STATUS_BOARD):** control-loop contract PROVEN · actor-owned production wiring NOT proven · cognitive-shader-driver/MailboxSoA thought NOT proven (the MUL gate is real; its qualia inputs are extractor-fed — "shader plug" wording retired in favor of "MUL-gate plug") · durability FAKE. **19 lib tests green** (was 14; +5 net new falsifiers), clippy+fmt clean. + ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — D-MBX-A6-P4c shader plug: the REAL MUL cognitive gate wired into the CognitiveWork seam - `lance-graph-supervisor::cycle_driver` gained the **shader plug** — the P4c thought body is no longer just a pluggable seam, it is the REAL MUL cognitive gate (mints NO decision logic; it composes `kanban_actor::mul_target` for the driver from two already-shipped contract primitives): diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index d8156727..0eed045e 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -964,7 +964,7 @@ Plan path: `.claude/plans/unified-soa-convergence-v1.md`. Handover `.claude/hand | D-MBX-A6-P3b | output overhaul: `StrategyOutcome{reliability, intended_move: Option}` carrier on `PlanInput.outcome`; StyleStrategy retires the dead-store `_reliability`, SURFACES reliability + a bootstrap intended move (Planning→CognitiveWork, owner 0, warden-BOOTSTRAP-OK) — plan still pure | lance-graph-planner | 130 | LOW | **In progress** | additive Option field (6 in-crate literals); UNBLOCKED (no mint, not OQ-11.7); deferred: compose thread-out + contract-promote + owner-consume; E-STRATEGY-OUTCOME-CARRIER-1 | | D-MBX-A6-P3c | owner-consume: `lance_graph_planner::owner_adapter` = the `Outcome → KanbanMove` bootstrap-rebind + ahead-cast adapter. `rebind_bootstrap` (mailbox 0/cycle 0 sentinel → live owner; refuses an already-owned move = no ownership theft) + `emit_bootstrap_intent` → `BatchWriter::cast(on_behalf = owner)`. Fire-and-forget (no ack/ledger/WAL/arbitration/callback); the move is the pre-write "parcel address", the lifecycle STEP stays post-write. Completes P3b's deferred `owner-consume`. | lance-graph-planner | 90 | LOW | **In PR** | 5 falsifiable probes (rebind 0→live anti-vacuity + no-theft + on-behalf cast + non-vacuous no-op silence); lance-free, builds without protoc. Persistence sink (drain→Lance 7 `mem_wal::WalAppender::append`) verified-but-gated (protoc missing + disk); knowledge doc `.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` | | D-MBX-A6-P3d | persistence sink ORDERING CORE + durable-witness reshape + temporal layer-1 (the POST-write half). `lance_graph_planner::persist_sink`: two clock domains (async `persist_cast` no-owner-borrow → `DurableReceipt`; sync `apply_durable_step` no-await → `try_advance_phase`). Crash-durability: `DurableWitness{owner,cast_id,cycle,paired_move}` CO-LOCATED with the SoA payload in one generation via `DurableWrite::append(&witness,&payload)`; `scan_witnesses(from)` bounded replay seam returning `LandedWitness{coordinate,witness}`; `recover_and_apply(owner,landed,applied_through)` replays the pending tail in **durable-log order** (`DurableCoordinate::log_order`, NOT the resettable `cast_id`) with a durable **watermark** for cyclic-safe idempotence, returning `Recovered{applied,watermark}`; `StalePhase` = corruption above the watermark (sync path: safe-to-drop stale). `temporal::{LocalCausalRow, local_trajectories, local_trajectory_of}` = layer-1 CAUSAL deinterlacing (global interleaved log → per-owner local chain), composing with the existing layer-2 epistemic projection. Durability proof = `DurableCoordinate` (opaque `seq`, API-honest), never `LanceVersion`. | lance-graph-planner | 175 | LOW | **Merged** | #878 (merged; reshaped in place to the cycle/WAL model = P3e — `E-THE-DURABLE-UNIT-IS-THE-CYCLE-NOT-THE-CAST-ONE-WAL-WRITE-PER-SWEEP-1`; + the §2 sparse-delta storage ruling `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`, RATIFIED/UNIMPLEMENTED); ordering/recovery CONTRACT probed (348 planner lib tests, clippy+fmt clean) — crash-durability NOT storage-proven (in-process fake, no real MemWAL/restart); review-hardened ×2 (Bugbot: cast_id-resets→durable-position order; Codex/CodeRabbit Critical: cyclic idempotence via durable watermark + negative control; cross-owner reject; concurrent-drain retryable receipt; bounded scan; contract-probe honesty); builds NO concrete `LanceShardSink`; generation-vs-per-cast seam (finding 5) surfaced for operator decision; `E-THE-PAIRED-MOVE-MUST-BE-DURABLE-CO-LOCATED-NOT-IN-MEMORY-ONLY-1` | -| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a–P4f Shipped (slice)** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** `collect_casts`+`seal_cycle` (drain `BatchWriter` → `persist_cycle` → one WAL write / one `DatasetVersion` + sparse `SealedTransition` set). **P4b** `apply_sealed_transitions` (iterate ONLY the sealed sparse set; one legal `try_advance_phase`/represented owner; unrepresented byte-identical; interim ≤1/owner/cycle → `deferred`; unknown → `missing`; reads NO dataset). **P4c** `run_cognitive_work` (owners entering CognitiveWork run a pluggable thought seam → `owner_adapter::emit_bootstrap_intent` casts the next intent write-on-behalf → next cycle) **+ shader plug** `shade_owner`/`run_cognitive_work_gated` (the REAL MUL gate: `contract::mul::i4_eval::gate_decision_i4(qualia,mantissa)` → `KanbanColumn::advance_on_gate` — Flow→forward, Block→Prune-where-legal, Hold→rest — packaged as a bootstrap-sentinel `StrategyOutcome`; `mul_target` composed for the driver, mints no decision logic; `qualia`/`mantissa` via a caller extractor bridging the deferred `MailboxSoaView::qualia()` seam — MailboxSoa contract UNCHANGED). **P4d** wait-free (an incomplete owner never blocks a completed one — no barrier). **P4e** `recover_fleet` (composes `persist_sink::recover_and_apply` per owner; per-owner watermark idempotence; partial-progress kept on error). **P4f** `CountingFleet` scale probe. `run_cycle` convenience. **14 lib tests** incl. the **64k/17 falsifier** (exactly 17 advance, 65 519 byte-identical, one WAL write, zero reads), P4c round-trip, the **shader-gate falsifier** (Flow-qualia owner casts + advances to Evaluation while a Hold-qualia owner rests at CognitiveWork — the gate discriminates, three distinct outputs Flow/Hold/Block for three inputs, plus absorbing-column no-successor), P4d wait-free, P4e idempotence + load-bearing-watermark negative control, P4f **O(dirty)-not-O(fleet)** (640 resolves over a 64k fleet); clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`StrategyOutcome`/`recover_and_apply`/`emit_bootstrap_intent`/`MailboxSoaOwner`). CognitiveWork thought body is now the REAL MUL gate (`shade_owner`) via the caller-supplied qualia extractor; the only remaining fake is the durability leg (contract-probe `WalSink`, until `LanceShardSink`). Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | +| D-MBX-A6-P4 | **Cycle loop-closure driver** (PLANNED): the seam that makes the merged `persist_sink` cycle/WAL bootstrap load-bearing at 64k. Closes `owners think over Vn → produce material updates emit sparse intents → planner collects/coalesces/seals (one WAL, Vn+1) + exposes the sealed paired-transition set → supervisor applies ONLY the sealed sparse transitions (represented owners advance one legal step; unrepresented owners byte-identical) → CognitiveWork runs the thought → owner_adapter casts next intent → loop`. **A DatasetVersion is global knowledge, NOT permission to advance every mailbox (sparse-cycle ruling).** Applied INLINE by the writer (no dataset re-read; NOT 64k async `drive_once`). Interim rule: ≤1 durable phase transition per owner per sealed cycle. Mints NO new types. Sub-deliverables P4a (drain+seal) / P4b (apply sealed sparse set) / P4c (CognitiveWork+cast round-trip) / P4d (wait-free emit) / P4e (recovery) / P4f (sparse routing scale 16k/64k, W2a-gated), probe-first. | lance-graph-supervisor | 400 | HIGH | **P4a–P4f Shipped (slice)** | `lance-graph-supervisor::cycle_driver` behind feature `cycle-driver` (optional one-way `lance-graph-planner` dep, verified acyclic; default build stays light). **P4a** `collect_casts`+`seal_cycle` (drain `BatchWriter` → `persist_cycle` → one WAL write / one `DatasetVersion` + sparse `SealedTransition` set). **P4b** `apply_sealed_transitions` (iterate ONLY the sealed sparse set; one legal `try_advance_phase`/represented owner; unrepresented byte-identical; interim ≤1/owner/cycle → `deferred`; unknown → `missing`; reads NO dataset). **P4c** `run_cognitive_work` (owners entering CognitiveWork run a pluggable thought seam → `owner_adapter::emit_bootstrap_intent` casts the next intent write-on-behalf → next cycle) **+ MUL-gate plug** `shade_owner`/`run_cognitive_work_gated[_over]` (the real MUL *gate* — `contract::mul::i4_eval::gate_decision_i4(qualia,mantissa)` → `KanbanColumn::advance_on_gate`, Flow→forward / Block→Prune-where-legal / Hold→rescheduled via `held_owners` — packaged as a bootstrap-sentinel `StrategyOutcome`; `mul_target` composed for the driver, mints no decision logic; **NOT the cognitive-shader-driver/MailboxSoA dispatch** — qualia/mantissa via a caller extractor bridging the deferred `MailboxSoaView::qualia()` seam, MailboxSoa contract UNCHANGED). **Review round (grain-of-salt, 2026-08-02):** retry-safe seal (`SealFailure` carries the byte-identical frozen cycle); restart-stable `stream_position` (= caller's durable `position_base` + CastId, `next_position_base` cursor — CastId alone was the P3d "cast_id is provenance only" trap); normal apply advances the per-owner recovery **watermark** with the phase (one rule with recovery — no replay/StalePhase after a crash); ≤1-move/owner enforced **pre-seal** (`HeldIntent`+`restage_held` — sealed set == applied set, recovery agrees; no seal-then-discard, no silent `moves.first()` truncation); mid-apply error returns the applied prefix `Err((partial,cause))`; Hold = reschedule not strand (`held_owners` + `_over` re-poll); `recover_fleet` partitions history once (O(history), not O(fleet×history)). **P4d** wait-free-at-the-cast-boundary (sequential pass; concurrent per-owner execution = the actor leg, honestly scoped) (an incomplete owner never blocks a completed one — no barrier). **P4e** `recover_fleet` (composes `persist_sink::recover_and_apply` per owner; per-owner watermark idempotence; partial-progress kept on error). **P4f** `CountingFleet` scale probe. `run_cycle` convenience. **19 lib tests** incl. the **64k/17 falsifier** (exactly 17 advance, 65 519 byte-identical, one WAL write, zero reads), failed-seal→byte-identical-retry, restart-stable positions across writer reconstruction, normal-apply-advances-watermark (crash → recovery replays NOTHING), held-move-lands-next-cycle + recovery-agrees, multi-move no-truncation, mid-apply prefix preservation, the **gate falsifier** (Flow casts + advances; Hold rescheduled and WOKEN on a later re-poll — the gate discriminates: three outputs for three inputs, absorbing-column no-successor), P4d unfinished-A-never-blocks-B (both represented), P4e idempotence + load-bearing-watermark negative control, P4f **O(dirty)-not-O(fleet)**; clippy+fmt clean. Mints NO domain types (reuses `SweepSlot`/`CycleFrame`/`KanbanMove`/`DatasetVersion`/`PersistError`/`StrategyOutcome`/`recover_and_apply`/`emit_bootstrap_intent`/`MailboxSoaOwner`). **Honesty ledger:** control-loop contract PROVEN (falsifiers over fakes) · actor-owned production wiring NOT proven (`MailboxFleet` HashMap = probe/registry fleet; bridging into `KanbanActor`-owned state open) · cognitive-shader-driver/SoA thought NOT proven (the gate is real, its inputs are extractor-fed) · durability FAKE (contract-probe `WalSink`, until `LanceShardSink`). Plan `.claude/plans/cycle-loop-closure-driver-v1.md`; `E-KANBANMOVE-IS-THE-PARCEL-ADDRESS-STEP-IS-THE-DELIVERY-SCAN-1` + `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1` + `E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1` | | D-MBX-A6-P3-M1 | `Tactic::requires() -> ThoughtMask` + `ThoughtField`/`ThoughtMask` (checklist-as-data keystone): 34 tactics declare their ThoughtCtx field-reads; `covered_by` = reliability-coverage gate | lance-graph-contract | 120 | LOW | **In PR** | #439; the panel-recalibrated keystone (extraction not construction); makes P1/P7/P11 derived; teeth-test asserts masks varied not stub | | D-CLS-FM | `class_view`: FieldMask(u64 presence) + ClassView meta-DTO resolver trait + ClassProjection (the class flies ABOVE the SoA; labels resolved late from OGIT cache, zero in the bytes) — extends ObjectView, reuses class_id | lance-graph-contract | 270 | LOW | **Shipped** | #441 D-CLS contract foundation; OD-gates ratified; presence!=semantics (C2); N3 stable positions; 3 teeth-tests | | D-CLS-RES | `class_resolver`: `RegistryClassView` impls `ClassView` over the live OntologyRegistry — the ontology-side 'parser' (class_id -> shape, DOLCE resolved LATE via classify_odoo from the cache URI, memoized over the O(n) registry scan) | lance-graph-ontology | 200 | LOW | **Shipped** | #441 D-CLS; makes the contract trait live; field-set supplied (D-CLS audit deferred); 4 teeth-tests | diff --git a/.claude/plans/cycle-loop-closure-driver-v1.md b/.claude/plans/cycle-loop-closure-driver-v1.md index 8e4d6715..f982213b 100644 --- a/.claude/plans/cycle-loop-closure-driver-v1.md +++ b/.claude/plans/cycle-loop-closure-driver-v1.md @@ -1,10 +1,17 @@ # cycle-loop-closure-driver-v1 — the loop-closure driver that makes the persist_sink cycle/WAL seam load-bearing -> **Status:** PLANNED / CONJECTURE — design only. The **CONTROL loop** this -> driver closes is the deliverable; the **durability leg stays the -> contract-probe fake** until the concrete `LanceShardSink` lands (the -> `compile+test green ≠ storage proven` Ladybug rule). Each claim below is -> probe-gated; nothing here is shipped. +> **Status:** IMPLEMENTED (slice, PR #879) — updated 2026-08-02 after the +> grain-of-salt review round. `lance-graph-supervisor::cycle_driver` (feature +> `cycle-driver`) ships P4a–P4f as **control-loop contract probes**: retry-safe +> seal, restart-stable stream positions (`position_base` durable cursor), +> watermark-coupled normal apply, pre-seal ≤1-move/owner partition +> (`HeldIntent`/`restage_held`), Hold-as-reschedule, prefix-preserving apply +> errors. **Honesty ledger:** control-loop contract PROVEN · actor-owned +> production wiring NOT proven (`MailboxFleet` HashMap = probe/registry fleet; +> `KanbanActor` bridging open) · cognitive-shader-driver/MailboxSoA thought NOT +> proven (the MUL gate is real, its qualia inputs extractor-fed) · **durability +> stays the contract-probe fake** until the concrete `LanceShardSink` lands (the +> `compile+test green ≠ storage proven` Ladybug rule). > **Date:** 2026-08-02. > **Scope:** documentation-only architectural ruling. Records the *missing > seam* — the driver that turns the already-merged `persist_sink` cycle/WAL @@ -352,12 +359,12 @@ W2a** and labelled a scale gate, while P4a…e are not. | Aspect | State | |---|---| -| `persist_sink` cycle/WAL seam (`persist_cycle` / `WalSink` / `recover_and_apply`) | **SHIPPED** (D-MBX-A6-P1…P3e) — but **ZERO production callers** (the loop is open) | +| `persist_sink` cycle/WAL seam (`persist_cycle` / `WalSink` / `recover_and_apply`) | **SHIPPED** (D-MBX-A6-P1…P3e) — first caller: `cycle_driver` (PR #879) | | `VersionScheduler` + `NextPhaseScheduler` (sync `on_version`) | **SHIPPED** contract (D-MBX-9-IN) | | `KanbanActor` + owner-apply (`try_advance_phase`) | **SHIPPED** (D-V3-W2b) | | `owner_adapter` + `BatchWriter` (Outcome → next-cycle cast) | **SHIPPED** (planner) | | `symbiont::kanban_loop::SymbiontBoard` (the shape-proving slice) | **SHIPPED** (D2) — `u32` tick placeholder for the real version | -| **CycleDriver** (P4a…f — closes seal→step→think→cast) | **PLANNED / CONJECTURE** — this plan; probe-gated | +| **CycleDriver** (P4a…f — closes seal→step→think→cast) | **IMPLEMENTED (slice, PR #879)** — 19 falsifiers green incl. retry-safe seal, restart-stable positions, watermark-coupled apply, pre-seal held-move partition, Hold-reschedule. Actor-owned wiring + shader/SoA thought + durability remain open (header ledger) | | Home = `lance-graph-supervisor` + new planner path-dep (fallback: planner) | **DECIDED** (§5.1) — verify no cycle via `cargo tree` | | Durability leg (concrete `LanceShardSink`, real crash durability) | **DEFERRED** — driver wires the contract-probe fake; control loop closes regardless | | Board-as-tenant owner-resolution (D-V3-W2a) | **GATED** — driver uses `phase()` today; P4f scale gate adopts the tenant column when W2a un-gates | diff --git a/crates/lance-graph-supervisor/src/cycle_driver.rs b/crates/lance-graph-supervisor/src/cycle_driver.rs index 927b5101..c4de6c6a 100644 --- a/crates/lance-graph-supervisor/src/cycle_driver.rs +++ b/crates/lance-graph-supervisor/src/cycle_driver.rs @@ -1,4 +1,4 @@ -//! P4a / P4b — the cycle loop-closure driver (supervisor-side). +//! P4 — the cycle loop-closure driver (supervisor-side). //! //! This is the seam that makes the merged `persist_sink` cycle/WAL bootstrap //! **load-bearing**. Before this, `persist_sink::persist_cycle` had zero @@ -11,7 +11,8 @@ //! paired-transition set (only the owners that actually cast a move). //! - **P4b** ([`apply_sealed_transitions`]): iterate **only** the sealed sparse //! transitions, resolve each owner in the fleet, apply **one** legal -//! `try_advance_phase`. **Every unrepresented owner is left byte-identical.** +//! `try_advance_phase` AND advance the owner's durable **recovery watermark** +//! in the same pass. **Every unrepresented owner is left byte-identical.** //! //! ## The load-bearing rule (E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1) //! @@ -22,24 +23,48 @@ //! globally-complete cycle physically sparse //! (`E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). //! -//! **Interim conservative rule:** at most **one** durable phase transition per -//! owner per sealed cycle. A second sealed transition for an already-advanced -//! owner is **deferred** (held for the next sealed horizon), not applied. +//! ## The ≤1-transition-per-owner rule is enforced BEFORE sealing //! -//! ## Ownership (D-MBX spine) +//! [`collect_casts`] partitions: the **first** intent move per owner (in cast +//! order) seals as that owner's `paired_move`; **every further move** — a second +//! move in the same cast, or a later same-owner cast — is returned as a +//! [`HeldIntent`] for the caller to re-stage into a FUTURE cycle +//! ([`restage_held`]). Nothing is sealed and then discarded: the sealed +//! transition set and the normally-applied transition set are the **same set**, +//! so crash recovery and normal operation agree. (`AppliedCycle::deferred` +//! survives only as a defence-in-depth counter for sealed inputs produced +//! elsewhere; cycles sealed through [`collect_casts`] keep it at 0.) //! -//! The supervisor is the **exclusive runtime owner** of the fleet; the planner -//! **decides** (produces the casts + the persistence contract). This driver lives -//! in the supervisor and depends **one-way** on the planner (planner never deps -//! supervisor — no cycle). It applies through `MailboxSoaOwner::try_advance_phase` -//! directly — no ractor message bus needed for P4a/P4b. +//! ## Ownership (D-MBX spine) — scope honesty //! -//! ## Honesty +//! The supervisor crate is the **runtime-owner side** of the spine; the planner +//! **decides** (produces the casts + the persistence contract). This driver +//! depends **one-way** on the planner (planner never deps supervisor — no +//! cycle). Per the ratified sparse-cycle ruling it applies **writer-fires-inline** +//! through `MailboxSoaOwner::try_advance_phase` — no ractor message bus, no +//! dataset re-read, NOT a 64k async `drive_once` fan. //! -//! The control loop closes here; the **durability leg stays the contract-probe -//! fake** until the concrete `LanceShardSink` lands (`compile+test green ≠ -//! storage proven`, the Ladybug rule). `apply_sealed_transitions` reads **no** -//! dataset — the version was already sealed by `seal_cycle`. +//! **What [`MailboxFleet`] is and is NOT:** it is the driver's owner-resolution +//! seam (and its `HashMap` impl the probe/registry fleet used by the tests). +//! It is **NOT** a claim of production supervisor ownership: the actor-owned +//! path (`kanban_actor::KanbanActor`, behind the `supervisor` feature, where +//! actor State IS the owner and mutation is serialized in the mailbox handler) +//! remains the production actor tree. Bridging the sealed sparse set into +//! actor-owned state (a `MailboxFleet` impl that resolves into actor state, or +//! delivery via the owner's mailbox) is **open, not shipped**. +//! +//! ## Honesty ledger (what is proven vs not) +//! +//! - **Control-loop contract: proven** (this module's falsifiers, over fakes). +//! - **Actor-owned production wiring: NOT proven** (see above). +//! - **cognitive-shader-driver / MailboxSoA thought: NOT proven** — the P4c +//! thought body here is the real **MUL gate** ([`shade_owner`] = +//! `gate_decision_i4` + `advance_on_gate`), but its qualia/mantissa inputs +//! come from a caller-supplied extractor, NOT from a live `MailboxSoA` / +//! shader-driver dispatch. A MedCare thought is real only when that path runs. +//! - **Durability: fake** — the WAL leg is the contract-probe `WalSink` fake +//! until the concrete `LanceShardSink` lands (`compile+test green ≠ storage +//! proven`, the Ladybug rule). use std::collections::{HashMap, HashSet}; @@ -60,11 +85,12 @@ use lance_graph_planner::traits::StrategyOutcome; /// One sealed paired transition — a member of the **sparse** set a sealed cycle /// publishes. Carries `stream_position` so P4b applies transitions in canonical -/// order (later-position wins the interim one-per-owner slot). Boring by design: -/// it says *which owner advances where*, nothing else. +/// order and advances the recovery watermark. Boring by design: it says *which +/// owner advances where*, nothing else. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SealedTransition { - /// The canonical order key inherited from the cast's `SweepSlot`. + /// The canonical durable order key (see [`collect_casts`]'s + /// `position_base` contract — restart-stable, NOT a raw `CastId`). pub stream_position: u64, /// The owner that produced this transition (== `mv.mailbox`). pub owner: MailboxId, @@ -81,7 +107,28 @@ pub struct SealedCycle { /// The one version this cycle sealed into. pub version: DatasetVersion, /// Only the owners that cast a `paired_move` — the sparse subset. + /// With the pre-seal ≤1-per-owner partition, at most one per owner. pub transitions: Vec, + /// The first unused stream position AFTER this cycle, computed over **all** + /// sealed slots (including no-move landings). The caller's next + /// `position_base` is `max(previous_base, next_position_base)` — carrying + /// the previous base forward covers the empty-cycle case (where this is 0). + pub next_position_base: u64, +} + +/// A seal that failed at the WAL — **retry-safe**: carries the complete frozen +/// input back to the caller, byte-identical, so a retry submits the SAME cycle. +/// No owner was mutated (apply never ran); the writer stays drained; the frozen +/// casts here are the single surviving copy of the cycle. +#[derive(Debug)] +pub struct SealFailure { + /// The frame the failed seal was submitted under (retry with the same one, + /// or a refreshed `base_version` after a fence conflict). + pub frame: CycleFrame, + /// The complete frozen cast set — resubmit via [`seal_cycle`]. + pub casts: Vec, + /// Why the WAL write failed. + pub cause: PersistError, } /// P4b output — the effect of applying a sealed cycle's sparse transition set. @@ -89,21 +136,50 @@ pub struct SealedCycle { pub struct AppliedCycle { /// The version whose sealed transitions were applied. pub version: DatasetVersion, - /// One move per **advanced** owner (distinct owners; interim ≤1 per cycle). + /// One move per **advanced** owner (distinct owners; ≤1 per cycle). pub applied: Vec, - /// Extra same-owner sealed transitions held for the next horizon (interim - /// one-transition-per-owner rule). + /// Defence-in-depth counter: same-owner extras in a sealed input NOT + /// produced by [`collect_casts`] (which enforces ≤1/owner pre-seal, so + /// cycles sealed through it keep this at 0). A non-zero value means the + /// sealed set and the applied set diverge — investigate the producer. pub deferred: usize, /// Sealed transitions whose owner is not registered in the fleet (a /// registration gap — surfaced as a count, not a crash; the durable move - /// stays in the log for a later recovery pass). + /// stays in the log and is replayed by [`recover_fleet`] once the owner + /// registers). pub missing: usize, } -/// A resolvable collection of mailbox owners — the supervisor's production fleet. -/// P4b resolves each sealed transition's owner through this; **unrepresented -/// owners are never resolved and never touched** (that is what keeps them -/// byte-identical). +/// An intent move held back by the pre-seal ≤1-per-owner partition — NOT sealed +/// this cycle, NOT lost: re-stage it into a future cycle via [`restage_held`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HeldIntent { + /// The owner whose extra move is held. + pub owner: MailboxId, + /// The move to re-stage next cycle. + pub mv: KanbanMove, +} + +/// P4a (drain) output: the slots to seal this cycle + the intents held for a +/// future cycle by the ≤1-per-owner partition. +#[derive(Debug)] +pub struct CollectedCasts { + /// One landing per drained cast (payloads always land; at most the first + /// per-owner move is paired). + pub slots: Vec, + /// Every intent move beyond the first per owner — nothing is silently + /// truncated; re-stage via [`restage_held`]. + pub held: Vec, +} + +/// A resolvable collection of mailbox owners — the driver's owner-resolution +/// seam. P4b resolves each sealed transition's owner through this; +/// **unrepresented owners are never resolved and never touched** (that is what +/// keeps them byte-identical). +/// +/// Scope honesty: the `HashMap` impl below is the probe/registry fleet. The +/// production actor-owned path (`KanbanActor` state) is NOT bridged yet — see +/// the module docs' ownership section. pub trait MailboxFleet { /// The concrete owner type this fleet holds. type Owner: MailboxSoaOwner; @@ -113,8 +189,8 @@ pub trait MailboxFleet { fn owner_mut(&mut self, id: MailboxId) -> Option<&mut Self::Owner>; } -/// A `HashMap` keyed by `MailboxId` is the simplest concrete fleet — used by the -/// production supervisor's owner registry and by the tests. +/// A `HashMap` keyed by `MailboxId` — the probe/registry fleet (tests + any +/// non-actor registry). NOT the production actor tree (see module docs). impl MailboxFleet for HashMap { type Owner = O; fn owner(&self, id: MailboxId) -> Option<&O> { @@ -126,38 +202,72 @@ impl MailboxFleet for HashMap { } /// **P4a (drain).** Map the fleet's staged `BatchWriter` casts into `SweepSlot`s -/// for one cycle. One slot per cast: `stream_position` = the monotonic `CastId` -/// (cast order), `paired_move` = the cast's **first** intended move (interim: one -/// intended move per cast), `row` resolved by the caller's `row_of`, `payload` = -/// the drained descriptor bytes. Draining clears the writer's payload staging. +/// for one cycle, enforcing the **≤1-move-per-owner** rule BEFORE sealing. +/// +/// - `stream_position = position_base + CastId` — **`position_base` is the +/// caller's durable cursor**, NOT 0 after a restart. `CastId` alone is a +/// resettable in-memory counter (a reconstructed writer restarts it at 0), +/// while `stream_position` is the cross-cycle recovery watermark: a position +/// that ever repeats below an owner's watermark is silently skipped by +/// `recover_and_apply`. Take `position_base` from durable state — the +/// previous [`SealedCycle::next_position_base`] (max'd with the prior base) +/// or a scan of the sealed log. The restart falsifier pins this. +/// - The first intent move per owner (cast order) becomes that owner's +/// `paired_move`. **Every further move** — same cast or a later same-owner +/// cast — is returned in [`CollectedCasts::held`], never silently dropped +/// and never sealed-then-ignored. +/// - Casts with no move are no-step landings (`paired_move = None`) — they +/// coalesce into the cycle image but contribute no transition. /// -/// Casts with no recorded move become no-step landings (`paired_move = None`) — -/// they still coalesce into the cycle image but contribute no transition. +/// Draining clears the writer's payload staging. #[must_use] pub fn collect_casts( writer: &mut BatchWriter>, cycle: CycleId, + position_base: u64, mut row_of: impl FnMut(MailboxId) -> u64, -) -> Vec { +) -> CollectedCasts { // Drain first (ends the &mut borrow), then read the intent moves immutably. let drained: Vec<_> = writer.drain_pending_payloads().collect(); - drained - .into_iter() - .filter_map(|(cast, payload)| { - let owner = writer.on_behalf_of(cast)?; - let paired_move = writer - .intent_moves(cast) - .and_then(|moves| moves.first().copied()); - Some(SweepSlot { - cycle, - stream_position: cast.0, - owner, - row: row_of(owner), - paired_move, - payload, - }) - }) - .collect() + let mut paired_owners: HashSet = HashSet::new(); + let mut slots = Vec::with_capacity(drained.len()); + let mut held = Vec::new(); + for (cast, payload) in drained { + let Some(owner) = writer.on_behalf_of(cast) else { + continue; + }; + let mut paired_move = None; + for &mv in writer.intent_moves(cast).unwrap_or(&[]) { + if paired_move.is_none() && !paired_owners.contains(&owner) { + paired_move = Some(mv); + paired_owners.insert(owner); + } else { + // ≤1/owner/cycle, enforced pre-seal: the extra move is HELD for + // a future cycle — not sealed, not discarded, not truncated. + held.push(HeldIntent { owner, mv }); + } + } + slots.push(SweepSlot { + cycle, + stream_position: position_base + cast.0, + owner, + row: row_of(owner), + paired_move, + payload, + }); + } + CollectedCasts { slots, held } +} + +/// Re-stage held intents ([`CollectedCasts::held`]) into the writer for the +/// NEXT cycle. The re-cast is intent-only (empty payload — the original cast's +/// payload already sealed with its cycle). Returns the number re-staged. +pub fn restage_held(writer: &mut BatchWriter>, held: Vec) -> usize { + let n = held.len(); + for h in held { + writer.cast(h.owner, vec![h.mv], Vec::new()); + } + n } /// **P4a (seal).** Freeze the collected casts into one cycle: read out the @@ -166,12 +276,17 @@ pub fn collect_casts( /// one `DatasetVersion`**. `persist_cycle`'s own guards still run /// (cross-owner-move reject, cast/frame cycle-match). The version is published /// before any owner advances (P4b). +/// +/// **Retry-safe:** on a WAL failure the complete frozen cast set is returned in +/// [`SealFailure`] — no owner was mutated, and a retry (`seal_cycle(sink, +/// failure.frame, failure.casts)`) submits the byte-identical cycle. The one +/// clone held until commit success is the price of that guarantee. pub async fn seal_cycle( sink: &S, frame: CycleFrame, casts: Vec, -) -> Result { - // Read the transitions out BEFORE `persist_cycle` takes ownership of `casts`. +) -> Result> { + // Read the transitions + next base out BEFORE `persist_cycle` takes ownership. let mut transitions: Vec = casts .iter() .filter_map(|s| { @@ -183,25 +298,43 @@ pub async fn seal_cycle( }) .collect(); transitions.sort_by_key(|t| t.stream_position); - let version = persist_cycle(sink, frame, casts).await?; - Ok(SealedCycle { - version, - transitions, - }) + let next_position_base = casts + .iter() + .map(|s| s.stream_position + 1) + .max() + .unwrap_or(0); + // Held until commit success — the price of a byte-identical retry. + let frozen = casts.clone(); + match persist_cycle(sink, frame, casts).await { + Ok(version) => Ok(SealedCycle { + version, + transitions, + next_position_base, + }), + Err(cause) => Err(Box::new(SealFailure { + frame, + casts: frozen, + cause, + })), + } } -/// **P4b.** Apply **only** the sealed sparse transition set to the fleet. Iterate -/// the transitions in canonical `stream_position` order; resolve each owner and -/// apply **one** legal `try_advance_phase`. **Every unrepresented owner is left -/// byte-identical** — it is never resolved, never touched. +/// **P4b.** Apply **only** the sealed sparse transition set to the fleet, and +/// advance each applied owner's durable **recovery watermark** in the same +/// pass — the phase transition and the applied-through watermark move together, +/// so a crash after normal apply does NOT cause [`recover_fleet`] to replay it. +/// +/// Iterate the transitions in canonical `stream_position` order; resolve each +/// owner and apply **one** legal `try_advance_phase`. **Every unrepresented +/// owner is left byte-identical** — never resolved, never touched. /// -/// Interim rule: at most one durable transition per owner per cycle — a second -/// sealed transition for an already-advanced owner is **deferred**. A sealed -/// move whose `from` no longer matches the owner's phase is corruption +/// A sealed move whose `from` no longer matches the owner's phase is corruption /// ([`PersistError::StalePhase`]); a cross-owner move is /// [`PersistError::OwnerMismatch`] (defence in depth — `persist_cycle` already -/// rejected it). On such an error the fleet is left mid-apply; re-drive from the -/// persisted watermark via `persist_sink::recover_and_apply`. +/// rejected it). On such an error the **applied prefix is returned** +/// (`Err((partial, cause))`, mirroring `recover_and_apply`'s contract) with its +/// watermarks already advanced — the caller persists the prefix's watermarks and +/// re-drives the tail from durable state. /// /// Reads **no** dataset: the version was sealed by [`seal_cycle`]; this is a pure /// in-memory fan over the sparse set (no `scan_sealed`, no `versions`, no @@ -209,7 +342,8 @@ pub async fn seal_cycle( pub fn apply_sealed_transitions( fleet: &mut F, sealed: &SealedCycle, -) -> Result { + watermarks: &mut HashMap>, +) -> Result { let mut ordered: Vec<&SealedTransition> = sealed.transitions.iter().collect(); ordered.sort_by_key(|t| t.stream_position); @@ -218,9 +352,16 @@ pub fn apply_sealed_transitions( let mut deferred = 0usize; let mut missing = 0usize; + let partial = |applied: Vec, deferred, missing| AppliedCycle { + version: sealed.version, + applied, + deferred, + missing, + }; + for t in ordered { - // Interim ≤1-per-owner: a later sealed move for an already-advanced owner - // waits for the next sealed horizon. + // Defence-in-depth: collect_casts enforces ≤1/owner pre-seal, so this + // only fires for sealed inputs produced elsewhere. if advanced.contains(&t.owner) { deferred += 1; continue; @@ -231,87 +372,147 @@ pub fn apply_sealed_transitions( }; // Defence in depth (persist_cycle already validated at seal time). if t.mv.mailbox != t.owner { - return Err(PersistError::OwnerMismatch { - move_owner: t.mv.mailbox, - landing_owner: t.owner, - }); + return Err(( + partial(applied, deferred, missing), + PersistError::OwnerMismatch { + move_owner: t.mv.mailbox, + landing_owner: t.owner, + }, + )); } // Corruption guard: the sealed move's `from` must match the owner's phase. if t.mv.from != owner.phase() { - return Err(PersistError::StalePhase { - owner_phase: owner.phase(), - move_from: t.mv.from, - }); + return Err(( + partial(applied, deferred, missing), + PersistError::StalePhase { + owner_phase: owner.phase(), + move_from: t.mv.from, + }, + )); + } + match owner.try_advance_phase(t.mv.to) { + Ok(step) => { + applied.push(step); + advanced.insert(t.owner); + // Phase transition + watermark advance TOGETHER (one owner state). + let wm = watermarks.entry(t.owner).or_insert(None); + if wm.is_none_or(|w| w < t.stream_position) { + *wm = Some(t.stream_position); + } + } + Err(e) => { + return Err(( + partial(applied, deferred, missing), + PersistError::Illegal(e), + )); + } } - let step = owner - .try_advance_phase(t.mv.to) - .map_err(PersistError::Illegal)?; - applied.push(step); - advanced.insert(t.owner); } - Ok(AppliedCycle { - version: sealed.version, - applied, - deferred, - missing, - }) + Ok(partial(applied, deferred, missing)) +} + +/// The full effect of one driven cycle ([`run_cycle`]). +#[derive(Debug)] +pub struct CycleOutcome { + /// The sealed cycle (version + sparse transitions + next position base). + pub sealed: SealedCycle, + /// The applied effect (advanced owners + counters), watermarks advanced. + pub applied: AppliedCycle, + /// Intents held by the ≤1/owner partition — re-stage via [`restage_held`]. + pub held: Vec, +} + +/// A [`run_cycle`] failure. +#[derive(Debug)] +pub enum CycleError { + /// The WAL commit failed — **no owner mutated**; the boxed [`SealFailure`] + /// carries the byte-identical frozen cycle for retry via [`seal_cycle`]. + Seal(Box), + /// A guard tripped mid-apply — the applied prefix (with its watermarks + /// already advanced) is preserved; re-drive the tail via [`recover_fleet`]. + Apply { + /// The applied prefix before the error. + partial: AppliedCycle, + /// The guard that tripped. + cause: PersistError, + }, } /// Convenience: run one full cycle — **P4a** (drain the writer + seal) then -/// **P4b** (apply the sparse set). The one seam a running loop calls per cycle. -/// Returns both the sealed cycle and the applied effect. +/// **P4b** (apply the sparse set + advance watermarks). The one seam a running +/// loop calls per cycle. +/// +/// `position_base` is the durable stream cursor (see [`collect_casts`]); +/// `watermarks` is the fleet's per-owner recovery watermark map, advanced in +/// place alongside the phases. On [`CycleError::Seal`] the frozen cycle is +/// retryable; on [`CycleError::Apply`] the applied prefix is preserved. pub async fn run_cycle( sink: &S, fleet: &mut F, writer: &mut BatchWriter>, frame: CycleFrame, + position_base: u64, + watermarks: &mut HashMap>, row_of: impl FnMut(MailboxId) -> u64, -) -> Result<(SealedCycle, AppliedCycle), PersistError> +) -> Result where S: WalSink, F: MailboxFleet, { - let casts = collect_casts(writer, frame.cycle, row_of); - let sealed = seal_cycle(sink, frame, casts).await?; - let applied = apply_sealed_transitions(fleet, &sealed)?; - Ok((sealed, applied)) + let collected = collect_casts(writer, frame.cycle, position_base, row_of); + let sealed = seal_cycle(sink, frame, collected.slots) + .await + .map_err(CycleError::Seal)?; + match apply_sealed_transitions(fleet, &sealed, watermarks) { + Ok(applied) => Ok(CycleOutcome { + sealed, + applied, + held: collected.held, + }), + Err((partial, cause)) => Err(CycleError::Apply { partial, cause }), + } } -/// **P4c.** For each owner that ENTERED `CognitiveWork` this cycle (an applied -/// move whose `to` is [`KanbanColumn::CognitiveWork`]), run the pluggable thought -/// body and route its Outcome into the NEXT cycle's casts via -/// `owner_adapter::emit_bootstrap_intent`. Returns the number of next-cycle -/// intents cast. -/// -/// The thought body is a **seam, not designed here** (§5.4 of the plan): `think` -/// is `FnMut(&Owner) -> Option<(StrategyOutcome, payload)>`. `None` = the thought -/// produced no next intent (the owner rests). The Outcome's `intended_move` must -/// be a **bootstrap sentinel** (`mailbox 0`) — `owner_adapter` rebinds it to the -/// live owner (no-theft) and casts it **write-on-behalf**, so the owner announces -/// where it is going and the next cycle collects it. This never mutates a mailbox -/// (the step is P4b, post-seal); it only stages the next intent. -pub fn run_cognitive_work( +/// P4c output — what a cognitive pass did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CognitiveWorkOutcome { + /// Next-cycle intents cast (one per owner whose thought produced one). + pub cast: usize, + /// Owners evaluated this pass that produced NO cast (a gate `Hold`, a + /// declined/unfinished thought, or a non-sentinel outcome). **A Hold is a + /// reschedule, not a strand:** feed these back into + /// [`run_cognitive_work_over`] / [`run_cognitive_work_gated_over`] on a + /// later cycle so a Held owner is evaluated again. + pub held_owners: Vec, +} + +/// Shared body: evaluate the thought seam over an explicit owner set. Only +/// owners currently in `CognitiveWork` are evaluated; each either casts a +/// next-cycle intent (via `owner_adapter::emit_bootstrap_intent`) or lands in +/// `held_owners` for a later pass. +fn cognitive_pass( fleet: &F, - applied: &AppliedCycle, + owners: impl IntoIterator, writer: &mut BatchWriter>, mut think: impl FnMut(&F::Owner) -> Option<(StrategyOutcome, Vec)>, -) -> usize +) -> CognitiveWorkOutcome where F: MailboxFleet, { - let mut cast_count = 0usize; - for mv in &applied.applied { - // Only owners that just entered CognitiveWork run the thought body. - if mv.to != KanbanColumn::CognitiveWork { - continue; - } - let Some(owner) = fleet.owner(mv.mailbox) else { + let mut cast = 0usize; + let mut held_owners = Vec::new(); + for id in owners { + let Some(owner) = fleet.owner(id) else { continue; }; + if owner.phase() != KanbanColumn::CognitiveWork { + continue; + } + let mut did_cast = false; if let Some((outcome, payload)) = think(owner) { - // owner_adapter rebinds the bootstrap sentinel to this owner + casts it - // write-on-behalf; a non-sentinel / no-intent outcome stages nothing. + // owner_adapter rebinds the bootstrap sentinel to this owner + casts + // it write-on-behalf; a non-sentinel / no-intent outcome stages nothing. if emit_bootstrap_intent( &outcome, owner.mailbox_id(), @@ -321,18 +522,76 @@ where ) .is_some() { - cast_count += 1; + did_cast = true; } } + if did_cast { + cast += 1; + } else { + held_owners.push(id); + } } - cast_count + CognitiveWorkOutcome { cast, held_owners } } -/// **The shader plug (P4c gate).** Run the real MUL cognitive gate over an owner -/// that just entered `CognitiveWork` and lower the decision to the owner's next -/// intended move — the "thinking" the P4c seam routes. This is -/// `kanban_actor::mul_target` composed for the driver: it mints **no** new -/// decision logic, it reuses the two shipped contract primitives. +/// **P4c.** For each owner that ENTERED `CognitiveWork` this cycle (an applied +/// move whose `to` is [`KanbanColumn::CognitiveWork`]), run the pluggable +/// thought body and route its Outcome into the NEXT cycle's casts via +/// `owner_adapter::emit_bootstrap_intent`. +/// +/// The thought body is a **seam**: `think` is +/// `FnMut(&Owner) -> Option<(StrategyOutcome, payload)>`. `None` = no next +/// intent this pass — the owner is returned in +/// [`CognitiveWorkOutcome::held_owners`] and MUST be re-evaluated on a later +/// cycle via [`run_cognitive_work_over`] (a rest is a reschedule, never a +/// strand). The Outcome's `intended_move` must be a **bootstrap sentinel** +/// (`mailbox 0`) — `owner_adapter` rebinds it to the live owner (no-theft) and +/// casts it **write-on-behalf**. This never mutates a mailbox (the step is P4b, +/// post-seal); it only stages the next intent. +/// +/// Execution is sequential within the pass (a synchronous loop) — the wait-free +/// property is at the **cast/cycle boundary** (an owner whose thought declines +/// or is unfinished never blocks a completed owner's cast), NOT intra-pass +/// concurrent execution. Concurrent per-owner thought execution belongs to the +/// actor leg (`kanban_actor`), not this driver. +pub fn run_cognitive_work( + fleet: &F, + applied: &AppliedCycle, + writer: &mut BatchWriter>, + think: impl FnMut(&F::Owner) -> Option<(StrategyOutcome, Vec)>, +) -> CognitiveWorkOutcome +where + F: MailboxFleet, +{ + let entered = applied + .applied + .iter() + .filter(|mv| mv.to == KanbanColumn::CognitiveWork) + .map(|mv| mv.mailbox) + .collect::>(); + cognitive_pass(fleet, entered, writer, think) +} + +/// **P4c re-poll.** Evaluate the thought seam over an EXPLICIT owner set — the +/// re-scheduling lane for owners a previous pass returned in +/// [`CognitiveWorkOutcome::held_owners`]. Owners no longer in `CognitiveWork` +/// are skipped. +pub fn run_cognitive_work_over( + fleet: &F, + owners: &[MailboxId], + writer: &mut BatchWriter>, + think: impl FnMut(&F::Owner) -> Option<(StrategyOutcome, Vec)>, +) -> CognitiveWorkOutcome +where + F: MailboxFleet, +{ + cognitive_pass(fleet, owners.iter().copied(), writer, think) +} + +/// **The MUL-gate plug (P4c gate).** Run the MUL cognitive gate over an owner in +/// `CognitiveWork` and lower the decision to the owner's next intended move. +/// This composes `kanban_actor::mul_target` for the driver — it mints **no** +/// new decision logic, reusing the two shipped contract primitives: /// /// 1. read the owner's *current* phase, /// 2. run [`gate_decision_i4`]`(qualia, mantissa)` — the i4 TrustTexture × FlowState @@ -341,18 +600,19 @@ where /// [`KanbanColumn::advance_on_gate`] (`Flow` → forward, `Block` → /// Prune-where-legal, `Hold` → rest). /// -/// The result is packaged as a **bootstrap sentinel** [`StrategyOutcome`] -/// (`mailbox 0`, `witness_chain_position 0`) so -/// `owner_adapter::emit_bootstrap_intent` rebinds it to the live owner and casts -/// it write-on-behalf — **no mailbox is mutated here** (the durable step is P4b, -/// next cycle). Returns `None` when the gate **Holds** (or yields no legal -/// successor): the owner rests this cycle and casts nothing. +/// **Scope honesty:** this is the real MUL *gate*, NOT the +/// `cognitive-shader-driver` / `MailboxSoA` dispatch path — no SoA columns are +/// read here, and a MedCare thought is proven only when that path runs. The +/// result is a **bootstrap sentinel** [`StrategyOutcome`] (`mailbox 0`, +/// `witness_chain_position 0`) for `owner_adapter` to rebind; `None` = the gate +/// **Holds** (or no legal successor) — the owner rests this pass and is +/// re-scheduled via [`CognitiveWorkOutcome::held_owners`]. /// -/// `qualia` + `mantissa` are supplied by the caller because `MailboxSoaView` does -/// **not yet** expose `qualia()` (deferred — `soa_view.rs` "add `fn qualia` when -/// the first consumer arrives"). P4c is that first consumer; until the trait -/// method lands, a fleet-specific extractor bridges the seam. This keeps the -/// MailboxSoa contract UNCHANGED — no trait redesign. +/// `qualia` + `mantissa` are supplied by the caller because `MailboxSoaView` +/// does not yet expose `qualia()` (deferred — `soa_view.rs` "add `fn qualia` +/// when the first consumer arrives"). P4c is that first consumer; until the +/// trait method lands, a fleet-specific extractor bridges the seam. This keeps +/// the MailboxSoa contract UNCHANGED — no trait redesign. #[must_use] pub fn shade_owner( owner: &O, @@ -376,20 +636,19 @@ pub fn shade_owner( }) } -/// **P4c with the real shader wired in.** Like [`run_cognitive_work`], but the -/// thought body IS the MUL cognitive gate ([`shade_owner`]) rather than a -/// caller-supplied Outcome. For each owner that just entered `CognitiveWork`, -/// `read_gate` extracts that owner's `(qualia, signed_mantissa, reliability, -/// payload)` — the qualia seam the deferred `MailboxSoaView::qualia()` will -/// eventually close — and the gate decides the next move. A **Hold** (or an owner -/// `read_gate` declines with `None`) casts nothing. Returns the number of -/// next-cycle intents cast. +/// **P4c with the MUL gate wired in.** Like [`run_cognitive_work`], but the +/// thought body IS the MUL cognitive gate ([`shade_owner`]). For each owner that +/// just entered `CognitiveWork`, `read_gate` extracts that owner's `(qualia, +/// signed_mantissa, reliability, payload)` — the qualia seam the deferred +/// `MailboxSoaView::qualia()` will eventually close — and the gate decides. A +/// **Hold** (or a declined `read_gate`) lands the owner in +/// [`CognitiveWorkOutcome::held_owners`] for re-scheduling. pub fn run_cognitive_work_gated( fleet: &F, applied: &AppliedCycle, writer: &mut BatchWriter>, mut read_gate: impl FnMut(&F::Owner) -> Option<(QualiaI4_16D, i8, f32, Vec)>, -) -> usize +) -> CognitiveWorkOutcome where F: MailboxFleet, { @@ -400,6 +659,24 @@ where }) } +/// **P4c gated re-poll.** [`run_cognitive_work_gated`] over an explicit owner +/// set — how a Held owner gets its gate re-evaluated on a later cycle. +pub fn run_cognitive_work_gated_over( + fleet: &F, + owners: &[MailboxId], + writer: &mut BatchWriter>, + mut read_gate: impl FnMut(&F::Owner) -> Option<(QualiaI4_16D, i8, f32, Vec)>, +) -> CognitiveWorkOutcome +where + F: MailboxFleet, +{ + run_cognitive_work_over(fleet, owners, writer, |owner| { + let (qualia, mantissa, reliability, payload) = read_gate(owner)?; + let outcome = shade_owner(owner, &qualia, mantissa, reliability)?; + Some((outcome, payload)) + }) +} + /// The effect of a [`recover_fleet`] pass. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FleetRecovery { @@ -409,12 +686,14 @@ pub struct FleetRecovery { pub owners_recovered: usize, } -/// **P4e.** Fleet-level crash recovery: scan the sealed landings once and replay +/// **P4e.** Fleet-level crash recovery: scan the sealed landings ONCE, partition +/// them per owner (one pass over history, not O(fleet × history)), and replay /// each owner's PENDING tail via `persist_sink::recover_and_apply`, idempotent /// with a **per-owner watermark**. Only unreplayed moves (above the owner's -/// watermark) are applied; already-applied moves are skipped; unrepresented -/// owners are untouched. `watermarks` is updated in place with the new per-owner -/// watermark to persist alongside the SoA phase. +/// watermark) are applied; already-applied moves — including those applied by +/// the NORMAL path, whose watermarks [`apply_sealed_transitions`] advanced — +/// are skipped; unrepresented owners are untouched. `watermarks` is updated in +/// place with the new per-owner watermark to persist alongside the SoA phase. /// /// On a mid-owner failure the partial progress is kept: the failing owner's /// watermark is still advanced for its applied prefix (per @@ -431,14 +710,21 @@ where F: MailboxFleet, { let sealed: Vec = sink.scan_sealed(None).await.map_err(PersistError::Write)?; + // Partition once: per-owner tails in stored order (O(history), then each + // owner replays only its own tail). + let mut by_owner: HashMap> = HashMap::new(); + for ls in sealed { + by_owner.entry(ls.slot.owner).or_default().push(ls); + } let mut total_applied = 0usize; let mut owners_recovered = 0usize; for &id in fleet_ids { let Some(owner) = fleet.owner_mut(id) else { continue; }; + let tail: &[LandedSlot] = by_owner.get(&id).map_or(&[], Vec::as_slice); let wm = watermarks.get(&id).copied().flatten(); - match recover_and_apply(owner, &sealed, wm) { + match recover_and_apply(owner, tail, wm) { Ok(rec) => { if !rec.applied.is_empty() { owners_recovered += 1; @@ -467,7 +753,7 @@ mod tests { use lance_graph_contract::kanban::{ExecTarget, KanbanColumn}; use lance_graph_contract::soa_view::MailboxSoaView; use lance_graph_planner::persist_sink::{DetachedCycleBatch, LandedSlot, WriteFailed}; - use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Mutex; // ── Minimal in-RAM owner (mirrors the persist_sink FakeOwner) ─────────────── @@ -530,7 +816,7 @@ mod tests { } } - // ── The WAL sink fake — counts writes AND reads (base-fenced) ──────────────── + // ── The WAL sink fake — counts writes AND reads (base-fenced, failable) ───── struct SealedRec { frame: CycleFrame, version: DatasetVersion, @@ -540,7 +826,8 @@ mod tests { sealed: Mutex>, next_version: AtomicU64, wal_writes: AtomicU64, - reads: AtomicU64, // scan_sealed + versions — MUST stay 0 across P4a+P4b + reads: AtomicU64, // scan_sealed + versions — MUST stay 0 across P4a+P4b + fail_next: AtomicBool, // injects ONE retryable WAL failure } impl FakeWalSink { fn new() -> Self { @@ -549,6 +836,7 @@ mod tests { next_version: AtomicU64::new(1), wal_writes: AtomicU64::new(0), reads: AtomicU64::new(0), + fail_next: AtomicBool::new(false), } } fn wal_writes(&self) -> u64 { @@ -557,6 +845,9 @@ mod tests { fn reads(&self) -> u64 { self.reads.load(Ordering::SeqCst) } + fn fail_next_commit(&self) { + self.fail_next.store(true, Ordering::SeqCst); + } } impl WalSink for FakeWalSink { async fn commit_cycle( @@ -564,6 +855,9 @@ mod tests { base: DatasetVersion, batch: DetachedCycleBatch, ) -> Result { + if self.fail_next.swap(false, Ordering::SeqCst) { + return Err(WriteFailed("injected retryable WAL failure".into())); + } let mut sealed = self.sealed.lock().unwrap(); let head = sealed.last().map_or(DatasetVersion(0), |s| s.version); if base != head { @@ -638,13 +932,18 @@ mod tests { let sink = FakeWalSink::new(); let owners: Vec = (0..100).collect(); let mut w = writer_with_moves(&owners); - let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); - assert_eq!(casts.len(), 100, "one slot per staged cast"); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + assert_eq!(collected.slots.len(), 100, "one slot per staged cast"); + assert!(collected.held.is_empty(), "one move each → nothing held"); assert_eq!(sink.wal_writes(), 0, "collecting writes no WAL"); - let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) - .await - .unwrap(); + let sealed = seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + collected.slots, + ) + .await + .unwrap(); assert_eq!(sink.wal_writes(), 1, "100 casts → exactly ONE WAL write"); assert_eq!(sealed.version, DatasetVersion(1), "→ exactly one version"); assert_eq!( @@ -652,9 +951,148 @@ mod tests { 100, "all 100 casts carried a move → sparse set = 100 here" ); + assert_eq!( + sealed.next_position_base, 100, + "next base = max position + 1" + ); // A second drain of the same writer is empty (payload staging cleared). - let again = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); - assert!(again.is_empty(), "drain cleared the writer's staging"); + let again = collect_casts(&mut w, CycleId(1), 0, u64::from); + assert!(again.slots.is_empty(), "drain cleared the writer's staging"); + } + + // ── RETRY FALSIFIER: a failed seal preserves the byte-identical cycle ─────── + #[tokio::test] + async fn failed_seal_preserves_the_frozen_cycle_for_byte_identical_retry() { + let sink = FakeWalSink::new(); + let mut fleet: HashMap = + HashMap::from([(9, FakeOwner::at(9, KanbanColumn::Planning))]); + let before = fleet.clone(); + let mut wm: HashMap> = HashMap::new(); + let mut w = writer_with_moves(&[9]); + + sink.fail_next_commit(); + let err = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, + u64::from, + ) + .await + .expect_err("injected WAL failure"); + + // Failed seal → zero owner mutation, and the writer stayed drained. + assert_eq!(fleet, before, "no owner advanced on a failed seal"); + let CycleError::Seal(failure) = err else { + panic!("expected a Seal failure"); + }; + assert_eq!(failure.casts.len(), 1, "the frozen cycle survived"); + let frozen_copy = failure.casts.clone(); + + // Retry submits the SAME frozen cycle → exactly one version lands. + let sealed = seal_cycle(&sink, failure.frame, failure.casts) + .await + .expect("retry succeeds"); + assert_eq!(sealed.version, DatasetVersion(1)); + assert_eq!(sink.wal_writes(), 1, "one successful WAL write total"); + // Byte-identical: what landed is exactly the frozen set. + let landed = sink.scan_sealed(None).await.unwrap(); + assert_eq!(landed.len(), 1); + assert_eq!(landed[0].slot, frozen_copy[0], "no cast lost or mutated"); + + let applied = apply_sealed_transitions(&mut fleet, &sealed, &mut wm).unwrap(); + assert_eq!(applied.applied.len(), 1, "owner advances exactly once"); + assert_eq!(fleet[&9].phase(), KanbanColumn::CognitiveWork); + } + + // ── RESTART FALSIFIER: stream positions stay monotonic across writer rebuilds ─ + #[tokio::test] + async fn restart_stable_stream_positions_survive_writer_reconstruction() { + let sink = FakeWalSink::new(); + let mut fleet: HashMap = + HashMap::from([(5, FakeOwner::at(5, KanbanColumn::Planning))]); + let mut wm: HashMap> = HashMap::new(); + + // Cycle 1 (base 0): owner 5 Planning→CognitiveWork at position 0. + let mut w1 = writer_with_moves(&[5]); + let c1 = collect_casts(&mut w1, CycleId(1), 0, u64::from); + let s1 = seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + c1.slots, + ) + .await + .unwrap(); + apply_sealed_transitions(&mut fleet, &s1, &mut wm).unwrap(); + assert_eq!(wm[&5], Some(0), "cycle-1 watermark at position 0"); + + // RESTART: the writer is RECONSTRUCTED (CastId restarts at 0). The + // durable cursor carries forward: base = s1.next_position_base. + let mut w2: BatchWriter> = BatchWriter::new(); + w2.cast( + 5, + vec![mv(5, KanbanColumn::CognitiveWork, KanbanColumn::Evaluation)], + vec![0xCD], + ); + let base2 = s1.next_position_base; + assert_eq!(base2, 1); + let c2 = collect_casts(&mut w2, CycleId(2), base2, u64::from); + assert_eq!( + c2.slots[0].stream_position, 1, + "restart-stable: NOT the raw CastId 0" + ); + seal_cycle( + &sink, + CycleFrame::new(CycleId(2), DatasetVersion(1)), + c2.slots, + ) + .await + .unwrap(); + + // Crash AFTER cycle 2 sealed but BEFORE it applied: recovery from the + // cycle-1 watermark must NOT skip the cycle-2 landing. (With the raw + // CastId scheme its position would be 0 ≤ watermark 0 → silently lost.) + let rec = recover_fleet(&sink, &mut fleet, &[5], &mut wm) + .await + .unwrap(); + assert_eq!(rec.total_applied, 1, "the later landing was replayed"); + assert_eq!(fleet[&5].phase(), KanbanColumn::Evaluation); + assert_eq!(wm[&5], Some(1)); + } + + // ── WATERMARK-COUPLING FALSIFIER: normal apply advances recovery watermarks ─ + #[tokio::test] + async fn normal_apply_advances_the_recovery_watermark_no_replay_after_crash() { + let sink = FakeWalSink::new(); + let mut fleet: HashMap = + HashMap::from([(5, FakeOwner::at(5, KanbanColumn::Planning))]); + let mut wm: HashMap> = HashMap::new(); + let mut w = writer_with_moves(&[5]); + + // Normal path: seal + apply. Watermark advances WITH the phase. + run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, + u64::from, + ) + .await + .unwrap(); + assert_eq!(fleet[&5].phase(), KanbanColumn::CognitiveWork); + assert_eq!(wm[&5], Some(0), "normal apply advanced the watermark"); + + // Crash + recovery with the SAME persisted watermarks: nothing replays, + // no StalePhase — the normal path and recovery share one rule. + let rec = recover_fleet(&sink, &mut fleet, &[5], &mut wm) + .await + .expect("recovery after a normal apply must not StalePhase-stall"); + assert_eq!(rec.total_applied, 0, "already-applied move NOT replayed"); + assert_eq!(fleet[&5].phase(), KanbanColumn::CognitiveWork); } // ── P4b HEADLINE FALSIFIER: 64k mailboxes / 17 sealed → exactly 17 advance ── @@ -669,26 +1107,32 @@ mod tests { .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) .collect(); let before = fleet.clone(); + let mut wm: HashMap> = HashMap::new(); let sink = FakeWalSink::new(); let mut w = writer_with_moves(&represented); - let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); - let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) - .await - .unwrap(); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + let sealed = seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + collected.slots, + ) + .await + .unwrap(); assert_eq!( sealed.transitions.len(), 17, "sparse sealed set = 17, not 64k" ); - let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + let applied = apply_sealed_transitions(&mut fleet, &sealed, &mut wm).unwrap(); // Exactly the 17 represented owners advanced. assert_eq!(applied.applied.len(), 17, "exactly 17 owners advanced"); assert_eq!(applied.deferred, 0); assert_eq!(applied.missing, 0); assert_eq!(applied.version, DatasetVersion(1)); + assert_eq!(wm.len(), 17, "exactly 17 watermarks advanced"); // Every represented owner is now at CognitiveWork (cycle bumped); every // OTHER owner is byte-identical to its pre-cycle state. @@ -731,19 +1175,24 @@ mod tests { .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) .collect(); let before = fleet.clone(); + let mut wm: HashMap> = HashMap::new(); let sink = FakeWalSink::new(); let mut w: BatchWriter> = BatchWriter::new(); for id in 0..1_000u32 { w.cast(id, vec![], vec![0x00]); // no move → no transition } - let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); - let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) - .await - .unwrap(); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + let sealed = seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + collected.slots, + ) + .await + .unwrap(); assert!(sealed.transitions.is_empty(), "no moves → empty sparse set"); - let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + let applied = apply_sealed_transitions(&mut fleet, &sealed, &mut wm).unwrap(); assert!( applied.applied.is_empty(), "a version alone advances nobody" @@ -752,13 +1201,14 @@ mod tests { assert_eq!(sink.wal_writes(), 1, "the cycle still sealed one version"); } - // ── P4b: interim one-transition-per-owner defers the rest ──────────────────── + // ── ≤1/owner is enforced BEFORE sealing: extras are HELD, never discarded ─── #[tokio::test] - async fn p4b_defers_a_second_transition_for_the_same_owner() { + async fn second_same_owner_move_is_held_pre_seal_and_lands_next_cycle() { let mut fleet: HashMap = HashMap::from([(42, FakeOwner::at(42, KanbanColumn::Planning))]); + let mut wm: HashMap> = HashMap::new(); let sink = FakeWalSink::new(); - // Two casts for owner 42 in one cycle. + // Two casts for owner 42 staged in one cycle. let mut w: BatchWriter> = BatchWriter::new(); w.cast( 42, @@ -774,23 +1224,80 @@ mod tests { )], vec![2], ); - let casts = collect_casts(&mut w, CycleId(1), |id| u64::from(id)); - let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) - .await - .unwrap(); - assert_eq!(sealed.transitions.len(), 2, "both cast a move"); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + // The partition happened PRE-SEAL: one paired transition, one held. + assert_eq!(collected.held.len(), 1, "the second move is HELD"); + assert_eq!(collected.held[0].mv.to, KanbanColumn::Evaluation); + assert_eq!( + collected.slots[1].paired_move, None, + "the extra cast still lands its payload, move-free" + ); - let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + let sealed = seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + collected.slots, + ) + .await + .unwrap(); assert_eq!( - applied.applied.len(), + sealed.transitions.len(), 1, - "≤1 durable transition per owner per cycle" + "sealed set == applied set (recovery and normal op agree)" + ); + let applied = apply_sealed_transitions(&mut fleet, &sealed, &mut wm).unwrap(); + assert_eq!(applied.applied.len(), 1); + assert_eq!(applied.deferred, 0, "nothing sealed was skipped"); + assert_eq!(fleet[&42].phase(), KanbanColumn::CognitiveWork); + + // Recovery from scratch applies EXACTLY the same set as normal op did. + let mut fresh = HashMap::from([(42, FakeOwner::at(42, KanbanColumn::Planning))]); + let mut wm2: HashMap> = HashMap::new(); + let rec = recover_fleet(&sink, &mut fresh, &[42], &mut wm2) + .await + .unwrap(); + assert_eq!(rec.total_applied, 1, "recovery applies the same ONE move"); + assert_eq!(fresh[&42].phase(), KanbanColumn::CognitiveWork); + + // The held intent re-stages and lands in the NEXT cycle. + assert_eq!(restage_held(&mut w, collected.held), 1); + let c2 = collect_casts(&mut w, CycleId(2), sealed.next_position_base, u64::from); + let s2 = seal_cycle( + &sink, + CycleFrame::new(CycleId(2), DatasetVersion(1)), + c2.slots, + ) + .await + .unwrap(); + let a2 = apply_sealed_transitions(&mut fleet, &s2, &mut wm).unwrap(); + assert_eq!(a2.applied.len(), 1, "the held move applied next cycle"); + assert_eq!(fleet[&42].phase(), KanbanColumn::Evaluation); + } + + // ── No silent truncation: a multi-move cast holds its extras too ──────────── + #[tokio::test] + async fn multi_move_single_cast_holds_extras_never_truncates() { + let mut w: BatchWriter> = BatchWriter::new(); + w.cast( + 7, + vec![ + mv(7, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + mv(7, KanbanColumn::CognitiveWork, KanbanColumn::Evaluation), + mv(7, KanbanColumn::Evaluation, KanbanColumn::Commit), + ], + vec![0xEE], ); - assert_eq!(applied.deferred, 1, "the second transition is deferred"); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + assert_eq!(collected.slots.len(), 1); assert_eq!( - fleet[&42].phase(), + collected.slots[0].paired_move.unwrap().to, KanbanColumn::CognitiveWork, - "owner advanced exactly one step (the first in stream order)" + "first move seals" + ); + assert_eq!( + collected.held.len(), + 2, + "the other two moves are HELD, not silently dropped" ); } @@ -800,6 +1307,7 @@ mod tests { // Owner is at CognitiveWork; the sealed move claims from=Planning. let mut fleet: HashMap = HashMap::from([(7, FakeOwner::at(7, KanbanColumn::CognitiveWork))]); + let mut wm: HashMap> = HashMap::new(); let sealed = SealedCycle { version: DatasetVersion(1), transitions: vec![SealedTransition { @@ -807,17 +1315,57 @@ mod tests { owner: 7, mv: mv(7, KanbanColumn::Planning, KanbanColumn::CognitiveWork), }], + next_position_base: 1, }; assert!(matches!( - apply_sealed_transitions(&mut fleet, &sealed), - Err(PersistError::StalePhase { .. }) + apply_sealed_transitions(&mut fleet, &sealed, &mut wm), + Err((_, PersistError::StalePhase { .. })) )); } + // ── P4b: a mid-apply error preserves the applied prefix + its watermarks ──── + #[tokio::test] + async fn p4b_mid_apply_error_returns_the_applied_prefix_with_watermarks() { + // Owner 1 has a good move; owner 2's sealed move is stale (corruption). + let mut fleet: HashMap = HashMap::from([ + (1, FakeOwner::at(1, KanbanColumn::Planning)), + (2, FakeOwner::at(2, KanbanColumn::CognitiveWork)), + ]); + let mut wm: HashMap> = HashMap::new(); + let sealed = SealedCycle { + version: DatasetVersion(1), + transitions: vec![ + SealedTransition { + stream_position: 0, + owner: 1, + mv: mv(1, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + }, + SealedTransition { + stream_position: 1, + owner: 2, + mv: mv(2, KanbanColumn::Planning, KanbanColumn::CognitiveWork), + }, + ], + next_position_base: 2, + }; + let (partial, cause) = + apply_sealed_transitions(&mut fleet, &sealed, &mut wm).expect_err("owner 2 is stale"); + assert!(matches!(cause, PersistError::StalePhase { .. })); + assert_eq!(partial.applied.len(), 1, "the applied prefix is preserved"); + assert_eq!(partial.applied[0].mailbox, 1); + assert_eq!( + wm.get(&1).copied().flatten(), + Some(0), + "the prefix's watermark was advanced before the error" + ); + assert_eq!(fleet[&1].phase(), KanbanColumn::CognitiveWork); + } + // ── P4b: a sealed move for an unregistered owner is counted, not a crash ──── #[tokio::test] async fn p4b_missing_owner_is_counted_not_applied() { let mut fleet: HashMap = HashMap::new(); // empty fleet + let mut wm: HashMap> = HashMap::new(); let sealed = SealedCycle { version: DatasetVersion(1), transitions: vec![SealedTransition { @@ -825,10 +1373,13 @@ mod tests { owner: 99, mv: mv(99, KanbanColumn::Planning, KanbanColumn::CognitiveWork), }], + next_position_base: 1, }; - let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + let applied = apply_sealed_transitions(&mut fleet, &sealed, &mut wm).unwrap(); assert_eq!(applied.missing, 1); assert!(applied.applied.is_empty()); + // The durable landing stays in the log: once the owner registers, + // recover_fleet replays it (the normal-path retry mechanism). } // ── End-to-end: run_cycle closes P4a→P4b in one call ──────────────────────── @@ -837,21 +1388,29 @@ mod tests { let mut fleet: HashMap = (0..10) .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) .collect(); + let mut wm: HashMap> = HashMap::new(); let sink = FakeWalSink::new(); let mut w = writer_with_moves(&[3, 7]); // only owners 3 and 7 produce a move - let (sealed, applied) = run_cycle( + let out = run_cycle( &sink, &mut fleet, &mut w, CycleFrame::new(CycleId(1), DatasetVersion(0)), - |id| u64::from(id), + 0, + &mut wm, + u64::from, ) .await .unwrap(); - assert_eq!(sealed.version, DatasetVersion(1)); - assert_eq!(applied.applied.len(), 2, "exactly owners 3 and 7 advanced"); + assert_eq!(out.sealed.version, DatasetVersion(1)); + assert_eq!( + out.applied.applied.len(), + 2, + "exactly owners 3 and 7 advanced" + ); + assert!(out.held.is_empty()); assert_eq!(fleet[&3].phase(), KanbanColumn::CognitiveWork); assert_eq!(fleet[&7].phase(), KanbanColumn::CognitiveWork); assert_eq!( @@ -880,14 +1439,17 @@ mod tests { let sink = FakeWalSink::new(); let mut fleet: HashMap = HashMap::from([(5, FakeOwner::at(5, KanbanColumn::Planning))]); + let mut wm: HashMap> = HashMap::new(); let mut w = writer_with_moves(&[5]); // Cycle 1: owner 5 casts Planning→CognitiveWork; the driver applies it. - let (_s1, applied1) = run_cycle( + let out1 = run_cycle( &sink, &mut fleet, &mut w, CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, u64::from, ) .await @@ -896,7 +1458,7 @@ mod tests { // P4c: owner 5 (now in CognitiveWork) thinks → intends CognitiveWork→Evaluation, // cast as a bootstrap sentinel into the writer for cycle 2. - let cast_count = run_cognitive_work(&fleet, &applied1, &mut w, |owner| { + let cw = run_cognitive_work(&fleet, &out1.applied, &mut w, |owner| { assert_eq!(owner.phase(), KanbanColumn::CognitiveWork); let outcome = StrategyOutcome { reliability: 0.9, @@ -907,64 +1469,93 @@ mod tests { }; Some((outcome, vec![0xCC])) }); - assert_eq!(cast_count, 1, "one next-cycle intent cast"); + assert_eq!(cw.cast, 1, "one next-cycle intent cast"); + assert!(cw.held_owners.is_empty()); // Cycle 2: the driver drains that cast → seals V2 → applies → owner 5 → Evaluation. - let (s2, applied2) = run_cycle( + let out2 = run_cycle( &sink, &mut fleet, &mut w, CycleFrame::new(CycleId(2), DatasetVersion(1)), + out1.sealed.next_position_base, + &mut wm, u64::from, ) .await .unwrap(); - assert_eq!(s2.version, DatasetVersion(2)); + assert_eq!(out2.sealed.version, DatasetVersion(2)); assert_eq!( - applied2.applied.len(), + out2.applied.applied.len(), 1, "the round-tripped intent advanced owner 5 one further step" ); assert_eq!(fleet[&5].phase(), KanbanColumn::Evaluation); } - // ── P4d FALSIFIER: an incomplete owner never blocks a completed one ───────── + // ── WAIT-FREE (strengthened): an unfinished thought never blocks a cast ───── #[tokio::test] - async fn p4d_an_incomplete_owner_never_blocks_a_completed_one() { - // Owner A(1) completes + casts; owner B(2) is "mid-thought" (no cast). + async fn p4d_an_unfinished_owner_never_blocks_a_completed_owners_cast() { + // BOTH owners are represented and BOTH enter CognitiveWork; A's thought + // stays unfinished (declines), B completes — B's cast lands regardless. let sink = FakeWalSink::new(); let mut fleet: HashMap = HashMap::from([ (1, FakeOwner::at(1, KanbanColumn::Planning)), (2, FakeOwner::at(2, KanbanColumn::Planning)), ]); - let mut w = writer_with_moves(&[1]); // ONLY A casts + let mut wm: HashMap> = HashMap::new(); + let mut w = writer_with_moves(&[1, 2]); - let (_s, applied) = run_cycle( + let out = run_cycle( &sink, &mut fleet, &mut w, CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, u64::from, ) .await .unwrap(); + assert_eq!(out.applied.applied.len(), 2, "both entered CognitiveWork"); - // A advanced without waiting for B; B is byte-identical. No barrier, no error. - assert_eq!(applied.applied.len(), 1); + let cw = run_cognitive_work(&fleet, &out.applied, &mut w, |owner| { + if owner.mailbox_id() == 1 { + None // A: unfinished / declines this pass + } else { + Some(( + StrategyOutcome { + reliability: 0.8, + intended_move: Some(sentinel( + KanbanColumn::CognitiveWork, + KanbanColumn::Evaluation, + )), + }, + vec![0xB2], + )) + } + }); + assert_eq!(cw.cast, 1, "B cast without waiting for A"); + assert_eq!(cw.held_owners, vec![1], "A is rescheduled, not lost"); + + // Cycle 2: B advances; A stays in CognitiveWork (unblocked, unfinished). + let out2 = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(2), DatasetVersion(1)), + out.sealed.next_position_base, + &mut wm, + u64::from, + ) + .await + .unwrap(); + assert_eq!(out2.applied.applied.len(), 1); + assert_eq!(fleet[&2].phase(), KanbanColumn::Evaluation, "B advanced"); assert_eq!( fleet[&1].phase(), KanbanColumn::CognitiveWork, - "A completed + advanced" - ); - assert_eq!( - fleet[&2].phase(), - KanbanColumn::Planning, - "B mid-thought never blocked A" - ); - assert_eq!( - fleet[&2].current_cycle(), - 0, - "B byte-identical — no neighbour wait" + "A unfinished, never a barrier for B" ); } @@ -974,10 +1565,14 @@ mod tests { // Seal a cycle with owner 5's Planning→CognitiveWork move (a durable landing). let sink = FakeWalSink::new(); let mut w = writer_with_moves(&[5]); - let casts = collect_casts(&mut w, CycleId(1), u64::from); - seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) - .await - .unwrap(); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + collected.slots, + ) + .await + .unwrap(); // Restart: a FRESH owner 5 at its pre-move phase, empty watermarks. let mut fleet: HashMap = @@ -1035,17 +1630,22 @@ mod tests { .map(|id| (id, FakeOwner::at(id, KanbanColumn::Planning))) .collect(); let mut fleet = CountingFleet { inner, resolves: 0 }; + let mut wm: HashMap> = HashMap::new(); let sink = FakeWalSink::new(); let represented: Vec = (0..DIRTY).map(|i| i * 100).collect(); let mut w = writer_with_moves(&represented); - let casts = collect_casts(&mut w, CycleId(1), u64::from); - let sealed = seal_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) - .await - .unwrap(); + let collected = collect_casts(&mut w, CycleId(1), 0, u64::from); + let sealed = seal_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + collected.slots, + ) + .await + .unwrap(); assert_eq!(sealed.transitions.len(), DIRTY as usize); - let applied = apply_sealed_transitions(&mut fleet, &sealed).unwrap(); + let applied = apply_sealed_transitions(&mut fleet, &sealed, &mut wm).unwrap(); assert_eq!( applied.applied.len(), DIRTY as usize, @@ -1062,7 +1662,7 @@ mod tests { ); } - // ── The shader plug (P4c gate) ────────────────────────────────────────────── + // ── The MUL-gate plug (P4c gate) ──────────────────────────────────────────── /// Flow qualia (warmth=4, groundedness=3, coherence=4, valence=2) — the same /// construction `kanban_actor::s2_driver_gate_advances_then_holds` uses: @@ -1127,34 +1727,35 @@ mod tests { ); } - // ── P4c GATED FALSIFIER: Flow-qualia thought → next cast → round-trip ──────── + // ── P4c GATED FALSIFIER: Flow casts + round-trips; Hold is RESCHEDULED ────── #[tokio::test] - async fn run_cognitive_work_gated_flow_casts_next_intent_hold_casts_nothing() { + async fn gated_flow_casts_hold_is_rescheduled_and_wakes_on_a_later_cycle() { let sink = FakeWalSink::new(); - // Owner 5 will FLOW (advances); owner 6 will HOLD (rests). + // Owner 5 will FLOW (advances); owner 6 will HOLD (rests, re-polled later). let mut fleet: HashMap = HashMap::from([ (5, FakeOwner::at(5, KanbanColumn::Planning)), (6, FakeOwner::at(6, KanbanColumn::Planning)), ]); + let mut wm: HashMap> = HashMap::new(); let mut w = writer_with_moves(&[5, 6]); // Cycle 1: both cast Planning→CognitiveWork; the driver applies both. - let (_s1, applied1) = run_cycle( + let out1 = run_cycle( &sink, &mut fleet, &mut w, CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, u64::from, ) .await .unwrap(); - assert_eq!(applied1.applied.len(), 2); - assert_eq!(fleet[&5].phase(), KanbanColumn::CognitiveWork); - assert_eq!(fleet[&6].phase(), KanbanColumn::CognitiveWork); + assert_eq!(out1.applied.applied.len(), 2); - // P4c: the REAL gate runs. Owner 5 gets Flow qualia → casts - // CognitiveWork→Evaluation. Owner 6 gets neutral qualia → Hold → no cast. - let cast_count = run_cognitive_work_gated(&fleet, &applied1, &mut w, |owner| { + // P4c: the REAL gate runs. Owner 5 Flows → casts CognitiveWork→Evaluation. + // Owner 6 gets neutral qualia → Hold → RESCHEDULED (held_owners), not lost. + let cw = run_cognitive_work_gated(&fleet, &out1.applied, &mut w, |owner| { let payload = vec![owner.mailbox_id() as u8]; if owner.mailbox_id() == 5 { Some((flow_qualia(), 4, 0.9, payload)) // FLOW @@ -1162,30 +1763,50 @@ mod tests { Some((QualiaI4_16D(0), 0, 0.5, payload)) // HOLD } }); - assert_eq!(cast_count, 1, "only the Flow owner cast a next intent"); + assert_eq!(cw.cast, 1, "only the Flow owner cast a next intent"); + assert_eq!(cw.held_owners, vec![6], "the Hold owner is RESCHEDULED"); - // Cycle 2: the driver drains that single cast → seals V2 → applies → - // owner 5 → Evaluation; owner 6 stayed at CognitiveWork (it Held). - let (s2, applied2) = run_cycle( + // Cycle 2: owner 5 advances to Evaluation; owner 6 rested this round. + let out2 = run_cycle( &sink, &mut fleet, &mut w, CycleFrame::new(CycleId(2), DatasetVersion(1)), + out1.sealed.next_position_base, + &mut wm, u64::from, ) .await .unwrap(); - assert_eq!(s2.version, DatasetVersion(2)); - assert_eq!(applied2.applied.len(), 1, "only the Flow owner advanced"); - assert_eq!( - fleet[&5].phase(), - KanbanColumn::Evaluation, - "Flow owner advanced one further step through the real gate" - ); + assert_eq!(fleet[&5].phase(), KanbanColumn::Evaluation); + assert_eq!(fleet[&6].phase(), KanbanColumn::CognitiveWork); + + // The Held owner WAKES: re-poll the held set with (now) Flow qualia — + // it casts, and the NEXT cycle advances it. No permanent strand. + let woken = run_cognitive_work_gated_over(&fleet, &cw.held_owners, &mut w, |owner| { + Some((flow_qualia(), 4, 0.7, vec![owner.mailbox_id() as u8])) + }); + assert_eq!(woken.cast, 1, "the Held owner cast on re-poll"); + assert!(woken.held_owners.is_empty()); + + let out3 = run_cycle( + &sink, + &mut fleet, + &mut w, + CycleFrame::new(CycleId(3), DatasetVersion(2)), + out2.sealed + .next_position_base + .max(out1.sealed.next_position_base), + &mut wm, + u64::from, + ) + .await + .unwrap(); + assert_eq!(out3.applied.applied.len(), 1); assert_eq!( fleet[&6].phase(), - KanbanColumn::CognitiveWork, - "Hold owner rested — the gate discriminates, it is not a constant" + KanbanColumn::Evaluation, + "a Hold is a reschedule, never a permanent strand" ); } }