From dbf9b6e0fd44fc44df37d15f8f5ce3c9b07fef68 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 24 Aug 2026 00:40:47 +0000 Subject: [PATCH 1/2] fix(daemon): stop capping how many projects can enrol Enrolling a fourth project failed with 'graph capacity budget exhausted (limit 8)', and once exhausted even a read-only projects list failed. Two graph owners per mounted project plus two profile-wide ones is exactly 8, so three projects filled the budget. Raises both ceilings to runaway guards instead of population bounds, and records why LRU cannot rescue this today: a mounted project holds its owner attachment for the life of the mount, so the eviction candidate search can never select it. Genuine elasticity needs idle-project hibernation, which is follow-up work. Co-Authored-By: Claude Opus 5 (1M context) --- crates/tracedecay-graph-db/src/registry.rs | 10 +++++++++ src/daemon/store_runtime/session_registry.rs | 21 +++++++++++++++++-- .../store_runtime/session_registry/mounts.rs | 9 +++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/registry.rs b/crates/tracedecay-graph-db/src/registry.rs index f7cb8e412..f3a46fa1e 100644 --- a/crates/tracedecay-graph-db/src/registry.rs +++ b/crates/tracedecay-graph-db/src/registry.rs @@ -1894,6 +1894,16 @@ fn rollback_retiring_under_lock( Ok(()) } +/// Reclaims one idle owner so `opening` can take its slot. +/// +/// A candidate must be `Ready` *and* unleased, and a mounted project never is: +/// the daemon holds its `GraphDbOwnerAttachmentV1` for as long as the project +/// stays mounted, so `is_unleased()` stays false and the search below cannot +/// select it. In practice that means this reclaims nothing for live projects +/// and the caller's `max_open` behaves as a population ceiling rather than a +/// working-set one. Making it a true LRU needs a way to hibernate an idle +/// project's owner and remount it on next access; until that exists, the +/// ceilings are sized as runaway guards instead. fn reserve_capacity_eviction( state: &mut RegistryState, max_open: usize, diff --git a/src/daemon/store_runtime/session_registry.rs b/src/daemon/store_runtime/session_registry.rs index 272b78f02..ae43197f1 100644 --- a/src/daemon/store_runtime/session_registry.rs +++ b/src/daemon/store_runtime/session_registry.rs @@ -55,8 +55,25 @@ use retained_hook_tasks::RetainedHookTasks; pub(crate) use code_graph::RetainedCodeGraphRuntimeV1; pub(crate) use profile_memory::open_user_memory_db; -const MAX_RETAINED_PROJECT_RUNTIME_OWNERS: usize = 8; -const MAX_RETAINED_REMOTE_NODE_OWNERS: usize = 8; +/// Sanity ceilings on concurrently mounted owners, not a bound on how many +/// projects a profile may enrol. +/// +/// These were 8, which is where enrolment actually stopped: a profile with +/// four projects was refused, and once the budget was exhausted even a +/// read-only `projects list` failed. Nothing about 8 was derived — the working +/// set of a developer with a few dozen checkouts is far above it. +/// +/// Sizing is honest about what does and does not bound this today. File +/// descriptors are not the constraint (the process ceiling is ~1M here and a +/// mount holds a handful). Resident memory *is* the constraint, and it is +/// currently ungoverned for these owners: no resident-memory gate covers a +/// mounted project runtime, and `reserve_capacity_eviction` cannot reclaim one +/// (see the note there), so a ceiling this high means residency grows with the +/// number of projects actually touched. That is the deliberate trade — a +/// refusal at 4 projects was the worse failure — and the real fix is idle +/// project hibernation, which is follow-up work. +const MAX_RETAINED_PROJECT_RUNTIME_OWNERS: usize = 4_096; +const MAX_RETAINED_REMOTE_NODE_OWNERS: usize = 4_096; struct SessionGraphOwnerV1 { graph: GraphDbOwnerAttachmentV1, diff --git a/src/daemon/store_runtime/session_registry/mounts.rs b/src/daemon/store_runtime/session_registry/mounts.rs index 077562af9..af53f736f 100644 --- a/src/daemon/store_runtime/session_registry/mounts.rs +++ b/src/daemon/store_runtime/session_registry/mounts.rs @@ -101,7 +101,14 @@ impl DaemonSessionRuntimeRegistryV1 { let graph_manifest_provider = Arc::new(super::code_graph_manifest::DaemonCodeGraphManifestProviderV1::default()); let graph_registry = GraphDbRegistry::new_with_manifest_provider( - GraphDbRegistryConfig { max_open: 8 }, + // Two graph owners per mounted project (its project shard and its + // session-relation shard) plus up to two profile-wide owners, so + // the previous 8 admitted exactly three projects before refusing + // the fourth. Sized here as a runaway guard well above any real + // working set rather than as a population bound; see + // MAX_RETAINED_PROJECT_RUNTIME_OWNERS for what actually bounds + // residency today. + GraphDbRegistryConfig { max_open: 8_192 }, graph_manifest_provider.clone(), ) .map_err(|error| { From 9123986a120ba0dba0268d3d35ef5e386805e3eb Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 24 Aug 2026 03:35:28 +0000 Subject: [PATCH 2/2] fix(daemon): derive the ceilings that admit the same population Two ceilings raised alongside the project ceiling were picked as round numbers rather than derived, and both were wrong in a way the advertised capacity hid. The graph registry slot ceiling was two slots short. Every mounted project holds its own graph owners on top of the two profile-wide ones, so 8,192 slots covered only 4,095 projects: the 4,096th failed inside GraphDbRegistry with a capacity budget error before ever reaching MAX_RETAINED_PROJECT_RUNTIME_OWNERS. It is now written as arithmetic over the project ceiling, the profile-wide owner count, and the graph owners one project can hold, so the two cannot drift apart again. The per-project figure is 3, not 2: RecoveryRequired retains the outgoing session owner alongside the candidate and Faulted retains both the retained and the faulted one, and none of them is reclaimable while the project stays mounted. The remote node owner ceiling disagreed with credential registration. Raising it to 4,096 left DaemonRemoteCredentialAuthorityV1 refusing register_storage at 128, and those two checks refuse at different points in the mount: admit_remote_node_owner refuses before anything is published, while register_storage refuses after mount_remote_node_storage has already published the runtime owner and provisioned the node's remote.db. Because mount_registered_remote_nodes remounts every discovered remote.db at daemon start with `?`, a rejected 129th provisioning left residue that failed every later start -- worse than the refusal it replaced. The mount ceiling now derives from the credential registry's MAX_REGISTERED_REMOTE_NODES, so the earlier, residue-free refusal is authoritative by construction. Unifying upward was rejected: the remote fleet size is a Remote Brain decision unrelated to how many local projects a profile enrols, and this PR raises only the latter. Rolling the mount back instead was also rejected. The in-memory owner is the easy half; the residue that wedges startup is the provisioned remote.db, and deleting a just-provisioned enrollment database is a destructive path this change does not need. Guards cover both, and each fails on the exact defect it names. With the shipped values they report "graph slot ceiling 8192 cannot admit 4096 projects: they need 12290 slots", "project 4096 is refused: 12287 of 8192 graph slots are already taken", and "remote mount ceiling 4096 exceeds the credential registry ceiling 128". A behavioural test in tracedecay-graph-db grounds the arithmetic in real registry behaviour: at the derived ceiling every owner attaches, and one slot short the final owner is refused with a capacity budget error, because a held attachment is never evictable. Co-Authored-By: Claude Opus 5 (1M context) --- crates/tracedecay-graph-db/src/registry.rs | 46 +++++++ src/daemon/remote_protocol.rs | 7 +- src/daemon/store_runtime/session_registry.rs | 130 +++++++++++++++++- .../store_runtime/session_registry/mounts.rs | 18 +-- 4 files changed, 188 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-graph-db/src/registry.rs b/crates/tracedecay-graph-db/src/registry.rs index f3a46fa1e..8e27fce8c 100644 --- a/crates/tracedecay-graph-db/src/registry.rs +++ b/crates/tracedecay-graph-db/src/registry.rs @@ -3215,4 +3215,50 @@ mod tests { assert_ne!(reopened.runtime_identity(), identity); assert!(reopened_attachment.shares_runtime_with(&reopened)); } + + /// `max_open` is a population ceiling for retained owners, so a caller + /// sizing it must count *every* owner that will be held, not only the + /// per-project ones. This reproduces the daemon's shape at small scale: + /// profile-wide owners are attached first, then each project's owners. + /// Sized as `profile_wide + per_project * projects` the last project + /// attaches; one slot short — the arithmetic slip that made the daemon's + /// advertised project capacity unreachable — it is refused with a capacity + /// budget error, because a held attachment is never evictable. + #[test] + fn a_capacity_ceiling_that_omits_profile_wide_owners_refuses_the_final_project() { + const PROFILE_WIDE_OWNERS: usize = 2; + const OWNERS_PER_PROJECT: usize = 3; + const PROJECTS: usize = 2; + let required = PROFILE_WIDE_OWNERS + OWNERS_PER_PROJECT * PROJECTS; + + let attach_all = |max_open: usize, + roots: &[TempDir]| + -> Result, GraphDbError> { + let registry = + GraphDbRegistry::new(GraphDbRegistryConfig { max_open }).unwrap(); + roots + .iter() + .enumerate() + .map(|(index, root)| { + registry.resolve_owner_attachment(owner_registration(registration_for( + root.path(), + &format!("project.capacity-ceiling-{index}"), + ))) + }) + .collect() + }; + + let roots: Vec = (0..required).map(|_| TempDir::new().unwrap()).collect(); + let attached = attach_all(required, &roots).expect("derived ceiling admits every owner"); + assert_eq!(attached.len(), required); + + let short_roots: Vec = (0..required).map(|_| TempDir::new().unwrap()).collect(); + assert_eq!( + attach_all(required - 1, &short_roots).unwrap_err(), + GraphDbError::BudgetExhausted { + kind: GraphBudgetKind::Capacity, + limit: (required - 1) as u64, + } + ); + } } diff --git a/src/daemon/remote_protocol.rs b/src/daemon/remote_protocol.rs index 4b8cf6146..4084765df 100644 --- a/src/daemon/remote_protocol.rs +++ b/src/daemon/remote_protocol.rs @@ -89,7 +89,12 @@ pub(super) fn remote_query_result_observation( ) } -const MAX_REGISTERED_REMOTE_NODES: usize = 128; +/// Canonical ceiling on Remote Brain nodes a profile may register. +/// +/// This is the single declared node-population ceiling: the session registry's +/// `MAX_RETAINED_REMOTE_NODE_OWNERS` derives from it so the mount admission +/// check, which refuses before anything is published, always refuses first. +pub(crate) const MAX_REGISTERED_REMOTE_NODES: usize = 128; const MAX_REGISTERED_REMOTE_CREDENTIALS: usize = 8_192; #[derive(Clone)] diff --git a/src/daemon/store_runtime/session_registry.rs b/src/daemon/store_runtime/session_registry.rs index ae43197f1..649b80f2b 100644 --- a/src/daemon/store_runtime/session_registry.rs +++ b/src/daemon/store_runtime/session_registry.rs @@ -55,10 +55,10 @@ use retained_hook_tasks::RetainedHookTasks; pub(crate) use code_graph::RetainedCodeGraphRuntimeV1; pub(crate) use profile_memory::open_user_memory_db; -/// Sanity ceilings on concurrently mounted owners, not a bound on how many -/// projects a profile may enrol. +/// Sanity ceiling on concurrently mounted project runtime owners, not a bound +/// on how many projects a profile may enrol. /// -/// These were 8, which is where enrolment actually stopped: a profile with +/// This was 8, which is where enrolment actually stopped: a profile with /// four projects was refused, and once the budget was exhausted even a /// read-only `projects list` failed. Nothing about 8 was derived — the working /// set of a developer with a few dozen checkouts is far above it. @@ -72,8 +72,55 @@ pub(crate) use profile_memory::open_user_memory_db; /// number of projects actually touched. That is the deliberate trade — a /// refusal at 4 projects was the worse failure — and the real fix is idle /// project hibernation, which is follow-up work. +/// +/// This is the *only* declared project-population ceiling. Every other ceiling +/// that has to admit the same projects is derived from it below, so the +/// advertised capacity cannot silently become unreachable again. const MAX_RETAINED_PROJECT_RUNTIME_OWNERS: usize = 4_096; -const MAX_RETAINED_REMOTE_NODE_OWNERS: usize = 4_096; + +/// Graph owners that exist once per profile rather than once per project: the +/// profile memory graph and the profile session-relation graph. Both are +/// retained for the life of the daemon, so they permanently occupy graph +/// registry slots that no project can use. +const PROFILE_WIDE_GRAPH_DB_OWNERS: usize = 2; + +/// Graph owners one mounted project can hold at the same time. +/// +/// A `Ready` project holds two: its memory graph and its session-relation +/// graph. While a session shard is being replaced the project holds a third — +/// `RecoveryRequired` retains the outgoing session owner alongside the +/// candidate, and `Faulted` retains both the retained and the faulted session +/// owner. `drain_retained_graph_owners_for_shutdown` enumerates exactly these +/// owners, and none of them is reclaimable while the project stays mounted. +const MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME: usize = 3; + +/// Graph registry slot ceiling, derived so the project ceiling above is +/// actually reachable. +/// +/// Written as arithmetic rather than a round literal on purpose. A hand-picked +/// 8,192 was two slots short of the advertised capacity: the profile-wide +/// owners take theirs first, so `2 + 2 * 4_095` already filled it and the +/// 4,096th project failed inside `GraphDbRegistry` before ever reaching +/// [`MAX_RETAINED_PROJECT_RUNTIME_OWNERS`]. Deriving it keeps the two ceilings +/// from drifting apart when either input changes. +pub(crate) const MAX_RETAINED_GRAPH_DB_OWNERS: usize = PROFILE_WIDE_GRAPH_DB_OWNERS + + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME * MAX_RETAINED_PROJECT_RUNTIME_OWNERS; + +/// Remote Brain node owner ceiling, taken from the credential registry rather +/// than declared independently. +/// +/// These two ceilings admit the same nodes, and they are not interchangeable +/// about *when* they refuse. `admit_remote_node_owner` refuses before anything +/// is published; `DaemonRemoteCredentialAuthorityV1::register_storage` refuses +/// after `mount_remote_node_storage` has already published the runtime owner +/// and provisioned the node's `remote.db`. Whenever this ceiling is the looser +/// of the two, the refusal lands on the later check and leaves a provisioned +/// database behind — and `mount_registered_remote_nodes` remounts every +/// discovered `remote.db` at startup with `?`, so the residue turns the next +/// daemon start into a hard failure. Binding this to the credential ceiling +/// keeps the earlier, residue-free refusal authoritative by construction. +const MAX_RETAINED_REMOTE_NODE_OWNERS: usize = + crate::daemon::remote_protocol::MAX_REGISTERED_REMOTE_NODES; struct SessionGraphOwnerV1 { graph: GraphDbOwnerAttachmentV1, @@ -2859,3 +2906,78 @@ mod project_memory_relation_graph_contract_tests; #[cfg(test)] mod tests; + +/// Guards on the ceilings above. Each one fails on the exact defect it names, +/// so raising one ceiling without the ceilings that have to admit the same +/// population is a test failure rather than a production refusal. +#[cfg(test)] +mod owner_capacity_ceiling_tests { + use super::*; + + /// The graph registry is what actually refuses a project mount, and the + /// profile-wide owners take their slots before any project does. A graph + /// ceiling that does not cover + /// `profile_wide + per_project * MAX_RETAINED_PROJECT_RUNTIME_OWNERS` + /// makes the advertised project capacity unreachable: the hand-picked + /// 8,192 left room for only 4,095 projects, so the 4,096th failed inside + /// `GraphDbRegistry` with a capacity budget error. + #[test] + fn graph_slot_ceiling_admits_every_project_the_project_ceiling_advertises() { + let required = PROFILE_WIDE_GRAPH_DB_OWNERS + .checked_add( + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME + .checked_mul(MAX_RETAINED_PROJECT_RUNTIME_OWNERS) + .expect("project graph slot demand must not overflow"), + ) + .expect("total graph slot demand must not overflow"); + + assert!( + MAX_RETAINED_GRAPH_DB_OWNERS >= required, + "graph slot ceiling {MAX_RETAINED_GRAPH_DB_OWNERS} cannot admit \ + {MAX_RETAINED_PROJECT_RUNTIME_OWNERS} projects: they need \ + {required} slots once the {PROFILE_WIDE_GRAPH_DB_OWNERS} \ + profile-wide owners have taken theirs" + ); + } + + /// The specific arithmetic the reviewer caught: with the profile-wide + /// owners mounted, the *last* project the ceiling advertises must still + /// find its own slots. Stated separately from the total so a ceiling that + /// is short by exactly the profile-wide owners fails here too. + #[test] + fn final_advertised_project_still_finds_graph_slots() { + let taken_before_final_project = PROFILE_WIDE_GRAPH_DB_OWNERS + + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME * (MAX_RETAINED_PROJECT_RUNTIME_OWNERS - 1); + + assert!( + taken_before_final_project + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME + <= MAX_RETAINED_GRAPH_DB_OWNERS, + "project {MAX_RETAINED_PROJECT_RUNTIME_OWNERS} is refused: \ + {taken_before_final_project} of {MAX_RETAINED_GRAPH_DB_OWNERS} graph \ + slots are already taken and it needs \ + {MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME} more" + ); + } + + /// Two ceilings admit the same Remote Brain nodes and they refuse at + /// different points in the mount. `admit_remote_node_owner` refuses before + /// the runtime owner is published and before `remote.db` is provisioned; + /// `register_storage` refuses after both. If the mount ceiling is the + /// looser of the two, a rejected provisioning leaves a provisioned + /// `remote.db` behind, and `mount_registered_remote_nodes` remounts every + /// discovered `remote.db` at daemon start with `?` — so the residue wedges + /// every subsequent start on the same node. + #[test] + fn remote_node_mount_ceiling_refuses_before_credential_registration_can() { + let credential_ceiling = crate::daemon::remote_protocol::MAX_REGISTERED_REMOTE_NODES; + + assert!( + MAX_RETAINED_REMOTE_NODE_OWNERS <= credential_ceiling, + "remote mount ceiling {MAX_RETAINED_REMOTE_NODE_OWNERS} exceeds the \ + credential registry ceiling {credential_ceiling}: node {} would publish \ + its owner and provision its database before being refused, and the \ + residue fails every later daemon start", + credential_ceiling + 1 + ); + } +} diff --git a/src/daemon/store_runtime/session_registry/mounts.rs b/src/daemon/store_runtime/session_registry/mounts.rs index af53f736f..69a0dc6c8 100644 --- a/src/daemon/store_runtime/session_registry/mounts.rs +++ b/src/daemon/store_runtime/session_registry/mounts.rs @@ -101,14 +101,16 @@ impl DaemonSessionRuntimeRegistryV1 { let graph_manifest_provider = Arc::new(super::code_graph_manifest::DaemonCodeGraphManifestProviderV1::default()); let graph_registry = GraphDbRegistry::new_with_manifest_provider( - // Two graph owners per mounted project (its project shard and its - // session-relation shard) plus up to two profile-wide owners, so - // the previous 8 admitted exactly three projects before refusing - // the fourth. Sized here as a runaway guard well above any real - // working set rather than as a population bound; see - // MAX_RETAINED_PROJECT_RUNTIME_OWNERS for what actually bounds - // residency today. - GraphDbRegistryConfig { max_open: 8_192 }, + // Derived from the project ceiling rather than hand-picked: the + // previous 8 admitted exactly three projects before refusing the + // fourth, because every mounted project holds its own graph owners + // on top of the profile-wide ones. See + // MAX_RETAINED_GRAPH_DB_OWNERS for the arithmetic and for why a + // round literal here is what made the advertised project capacity + // unreachable. + GraphDbRegistryConfig { + max_open: super::MAX_RETAINED_GRAPH_DB_OWNERS, + }, graph_manifest_provider.clone(), ) .map_err(|error| {