diff --git a/crates/tracedecay-graph-db/src/registry.rs b/crates/tracedecay-graph-db/src/registry.rs index f7cb8e412..8e27fce8c 100644 --- a/crates/tracedecay-graph-db/src/registry.rs +++ b/crates/tracedecay-graph-db/src/registry.rs @@ -1894,6 +1894,16 @@ fn rollback_retiring_under_lock( Ok(()) } +/// Reclaims one idle owner so `opening` can take its slot. +/// +/// A candidate must be `Ready` *and* unleased, and a mounted project never is: +/// the daemon holds its `GraphDbOwnerAttachmentV1` for as long as the project +/// stays mounted, so `is_unleased()` stays false and the search below cannot +/// select it. In practice that means this reclaims nothing for live projects +/// and the caller's `max_open` behaves as a population ceiling rather than a +/// working-set one. Making it a true LRU needs a way to hibernate an idle +/// project's owner and remount it on next access; until that exists, the +/// ceilings are sized as runaway guards instead. fn reserve_capacity_eviction( state: &mut RegistryState, max_open: usize, @@ -3205,4 +3215,50 @@ mod tests { assert_ne!(reopened.runtime_identity(), identity); assert!(reopened_attachment.shares_runtime_with(&reopened)); } + + /// `max_open` is a population ceiling for retained owners, so a caller + /// sizing it must count *every* owner that will be held, not only the + /// per-project ones. This reproduces the daemon's shape at small scale: + /// profile-wide owners are attached first, then each project's owners. + /// Sized as `profile_wide + per_project * projects` the last project + /// attaches; one slot short — the arithmetic slip that made the daemon's + /// advertised project capacity unreachable — it is refused with a capacity + /// budget error, because a held attachment is never evictable. + #[test] + fn a_capacity_ceiling_that_omits_profile_wide_owners_refuses_the_final_project() { + const PROFILE_WIDE_OWNERS: usize = 2; + const OWNERS_PER_PROJECT: usize = 3; + const PROJECTS: usize = 2; + let required = PROFILE_WIDE_OWNERS + OWNERS_PER_PROJECT * PROJECTS; + + let attach_all = |max_open: usize, + roots: &[TempDir]| + -> Result, GraphDbError> { + let registry = + GraphDbRegistry::new(GraphDbRegistryConfig { max_open }).unwrap(); + roots + .iter() + .enumerate() + .map(|(index, root)| { + registry.resolve_owner_attachment(owner_registration(registration_for( + root.path(), + &format!("project.capacity-ceiling-{index}"), + ))) + }) + .collect() + }; + + let roots: Vec = (0..required).map(|_| TempDir::new().unwrap()).collect(); + let attached = attach_all(required, &roots).expect("derived ceiling admits every owner"); + assert_eq!(attached.len(), required); + + let short_roots: Vec = (0..required).map(|_| TempDir::new().unwrap()).collect(); + assert_eq!( + attach_all(required - 1, &short_roots).unwrap_err(), + GraphDbError::BudgetExhausted { + kind: GraphBudgetKind::Capacity, + limit: (required - 1) as u64, + } + ); + } } diff --git a/src/daemon/remote_protocol.rs b/src/daemon/remote_protocol.rs index 4b8cf6146..4084765df 100644 --- a/src/daemon/remote_protocol.rs +++ b/src/daemon/remote_protocol.rs @@ -89,7 +89,12 @@ pub(super) fn remote_query_result_observation( ) } -const MAX_REGISTERED_REMOTE_NODES: usize = 128; +/// Canonical ceiling on Remote Brain nodes a profile may register. +/// +/// This is the single declared node-population ceiling: the session registry's +/// `MAX_RETAINED_REMOTE_NODE_OWNERS` derives from it so the mount admission +/// check, which refuses before anything is published, always refuses first. +pub(crate) const MAX_REGISTERED_REMOTE_NODES: usize = 128; const MAX_REGISTERED_REMOTE_CREDENTIALS: usize = 8_192; #[derive(Clone)] diff --git a/src/daemon/store_runtime/session_registry.rs b/src/daemon/store_runtime/session_registry.rs index 272b78f02..649b80f2b 100644 --- a/src/daemon/store_runtime/session_registry.rs +++ b/src/daemon/store_runtime/session_registry.rs @@ -55,8 +55,72 @@ use retained_hook_tasks::RetainedHookTasks; pub(crate) use code_graph::RetainedCodeGraphRuntimeV1; pub(crate) use profile_memory::open_user_memory_db; -const MAX_RETAINED_PROJECT_RUNTIME_OWNERS: usize = 8; -const MAX_RETAINED_REMOTE_NODE_OWNERS: usize = 8; +/// Sanity ceiling on concurrently mounted project runtime owners, not a bound +/// on how many projects a profile may enrol. +/// +/// This was 8, which is where enrolment actually stopped: a profile with +/// four projects was refused, and once the budget was exhausted even a +/// read-only `projects list` failed. Nothing about 8 was derived — the working +/// set of a developer with a few dozen checkouts is far above it. +/// +/// Sizing is honest about what does and does not bound this today. File +/// descriptors are not the constraint (the process ceiling is ~1M here and a +/// mount holds a handful). Resident memory *is* the constraint, and it is +/// currently ungoverned for these owners: no resident-memory gate covers a +/// mounted project runtime, and `reserve_capacity_eviction` cannot reclaim one +/// (see the note there), so a ceiling this high means residency grows with the +/// number of projects actually touched. That is the deliberate trade — a +/// refusal at 4 projects was the worse failure — and the real fix is idle +/// project hibernation, which is follow-up work. +/// +/// This is the *only* declared project-population ceiling. Every other ceiling +/// that has to admit the same projects is derived from it below, so the +/// advertised capacity cannot silently become unreachable again. +const MAX_RETAINED_PROJECT_RUNTIME_OWNERS: usize = 4_096; + +/// Graph owners that exist once per profile rather than once per project: the +/// profile memory graph and the profile session-relation graph. Both are +/// retained for the life of the daemon, so they permanently occupy graph +/// registry slots that no project can use. +const PROFILE_WIDE_GRAPH_DB_OWNERS: usize = 2; + +/// Graph owners one mounted project can hold at the same time. +/// +/// A `Ready` project holds two: its memory graph and its session-relation +/// graph. While a session shard is being replaced the project holds a third — +/// `RecoveryRequired` retains the outgoing session owner alongside the +/// candidate, and `Faulted` retains both the retained and the faulted session +/// owner. `drain_retained_graph_owners_for_shutdown` enumerates exactly these +/// owners, and none of them is reclaimable while the project stays mounted. +const MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME: usize = 3; + +/// Graph registry slot ceiling, derived so the project ceiling above is +/// actually reachable. +/// +/// Written as arithmetic rather than a round literal on purpose. A hand-picked +/// 8,192 was two slots short of the advertised capacity: the profile-wide +/// owners take theirs first, so `2 + 2 * 4_095` already filled it and the +/// 4,096th project failed inside `GraphDbRegistry` before ever reaching +/// [`MAX_RETAINED_PROJECT_RUNTIME_OWNERS`]. Deriving it keeps the two ceilings +/// from drifting apart when either input changes. +pub(crate) const MAX_RETAINED_GRAPH_DB_OWNERS: usize = PROFILE_WIDE_GRAPH_DB_OWNERS + + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME * MAX_RETAINED_PROJECT_RUNTIME_OWNERS; + +/// Remote Brain node owner ceiling, taken from the credential registry rather +/// than declared independently. +/// +/// These two ceilings admit the same nodes, and they are not interchangeable +/// about *when* they refuse. `admit_remote_node_owner` refuses before anything +/// is published; `DaemonRemoteCredentialAuthorityV1::register_storage` refuses +/// after `mount_remote_node_storage` has already published the runtime owner +/// and provisioned the node's `remote.db`. Whenever this ceiling is the looser +/// of the two, the refusal lands on the later check and leaves a provisioned +/// database behind — and `mount_registered_remote_nodes` remounts every +/// discovered `remote.db` at startup with `?`, so the residue turns the next +/// daemon start into a hard failure. Binding this to the credential ceiling +/// keeps the earlier, residue-free refusal authoritative by construction. +const MAX_RETAINED_REMOTE_NODE_OWNERS: usize = + crate::daemon::remote_protocol::MAX_REGISTERED_REMOTE_NODES; struct SessionGraphOwnerV1 { graph: GraphDbOwnerAttachmentV1, @@ -2842,3 +2906,78 @@ mod project_memory_relation_graph_contract_tests; #[cfg(test)] mod tests; + +/// Guards on the ceilings above. Each one fails on the exact defect it names, +/// so raising one ceiling without the ceilings that have to admit the same +/// population is a test failure rather than a production refusal. +#[cfg(test)] +mod owner_capacity_ceiling_tests { + use super::*; + + /// The graph registry is what actually refuses a project mount, and the + /// profile-wide owners take their slots before any project does. A graph + /// ceiling that does not cover + /// `profile_wide + per_project * MAX_RETAINED_PROJECT_RUNTIME_OWNERS` + /// makes the advertised project capacity unreachable: the hand-picked + /// 8,192 left room for only 4,095 projects, so the 4,096th failed inside + /// `GraphDbRegistry` with a capacity budget error. + #[test] + fn graph_slot_ceiling_admits_every_project_the_project_ceiling_advertises() { + let required = PROFILE_WIDE_GRAPH_DB_OWNERS + .checked_add( + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME + .checked_mul(MAX_RETAINED_PROJECT_RUNTIME_OWNERS) + .expect("project graph slot demand must not overflow"), + ) + .expect("total graph slot demand must not overflow"); + + assert!( + MAX_RETAINED_GRAPH_DB_OWNERS >= required, + "graph slot ceiling {MAX_RETAINED_GRAPH_DB_OWNERS} cannot admit \ + {MAX_RETAINED_PROJECT_RUNTIME_OWNERS} projects: they need \ + {required} slots once the {PROFILE_WIDE_GRAPH_DB_OWNERS} \ + profile-wide owners have taken theirs" + ); + } + + /// The specific arithmetic the reviewer caught: with the profile-wide + /// owners mounted, the *last* project the ceiling advertises must still + /// find its own slots. Stated separately from the total so a ceiling that + /// is short by exactly the profile-wide owners fails here too. + #[test] + fn final_advertised_project_still_finds_graph_slots() { + let taken_before_final_project = PROFILE_WIDE_GRAPH_DB_OWNERS + + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME * (MAX_RETAINED_PROJECT_RUNTIME_OWNERS - 1); + + assert!( + taken_before_final_project + MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME + <= MAX_RETAINED_GRAPH_DB_OWNERS, + "project {MAX_RETAINED_PROJECT_RUNTIME_OWNERS} is refused: \ + {taken_before_final_project} of {MAX_RETAINED_GRAPH_DB_OWNERS} graph \ + slots are already taken and it needs \ + {MAX_GRAPH_DB_OWNERS_PER_PROJECT_RUNTIME} more" + ); + } + + /// Two ceilings admit the same Remote Brain nodes and they refuse at + /// different points in the mount. `admit_remote_node_owner` refuses before + /// the runtime owner is published and before `remote.db` is provisioned; + /// `register_storage` refuses after both. If the mount ceiling is the + /// looser of the two, a rejected provisioning leaves a provisioned + /// `remote.db` behind, and `mount_registered_remote_nodes` remounts every + /// discovered `remote.db` at daemon start with `?` — so the residue wedges + /// every subsequent start on the same node. + #[test] + fn remote_node_mount_ceiling_refuses_before_credential_registration_can() { + let credential_ceiling = crate::daemon::remote_protocol::MAX_REGISTERED_REMOTE_NODES; + + assert!( + MAX_RETAINED_REMOTE_NODE_OWNERS <= credential_ceiling, + "remote mount ceiling {MAX_RETAINED_REMOTE_NODE_OWNERS} exceeds the \ + credential registry ceiling {credential_ceiling}: node {} would publish \ + its owner and provision its database before being refused, and the \ + residue fails every later daemon start", + credential_ceiling + 1 + ); + } +} diff --git a/src/daemon/store_runtime/session_registry/mounts.rs b/src/daemon/store_runtime/session_registry/mounts.rs index 077562af9..69a0dc6c8 100644 --- a/src/daemon/store_runtime/session_registry/mounts.rs +++ b/src/daemon/store_runtime/session_registry/mounts.rs @@ -101,7 +101,16 @@ impl DaemonSessionRuntimeRegistryV1 { let graph_manifest_provider = Arc::new(super::code_graph_manifest::DaemonCodeGraphManifestProviderV1::default()); let graph_registry = GraphDbRegistry::new_with_manifest_provider( - GraphDbRegistryConfig { max_open: 8 }, + // Derived from the project ceiling rather than hand-picked: the + // previous 8 admitted exactly three projects before refusing the + // fourth, because every mounted project holds its own graph owners + // on top of the profile-wide ones. See + // MAX_RETAINED_GRAPH_DB_OWNERS for the arithmetic and for why a + // round literal here is what made the advertised project capacity + // unreachable. + GraphDbRegistryConfig { + max_open: super::MAX_RETAINED_GRAPH_DB_OWNERS, + }, graph_manifest_provider.clone(), ) .map_err(|error| {